diff --git "a/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log" "b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log" @@ -0,0 +1,41260 @@ +2024-03-12 14:28:55,064 INFO StreamThr :452226 [internal.py:wandb_internal():86] W&B internal server running at pid: 452226, started at: 2024-03-12 14:28:55.062357 +2024-03-12 14:28:55,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:28:55,078 INFO WriterThread:452226 [datastore.py:open_for_write():87] open: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb +2024-03-12 14:28:55,080 DEBUG SenderThread:452226 [sender.py:send():382] send: header +2024-03-12 14:28:55,098 DEBUG SenderThread:452226 [sender.py:send():382] send: run +2024-03-12 14:28:55,575 INFO SenderThread:452226 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files +2024-03-12 14:28:55,575 INFO SenderThread:452226 [sender.py:_start_run_threads():1136] run started: srovbvi2 with start time 1710253735.064287 +2024-03-12 14:28:55,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:28:55,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:28:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:28:55,654 DEBUG HandlerThread:452226 [system_info.py:__init__():27] System info init +2024-03-12 14:28:55,654 DEBUG HandlerThread:452226 [system_info.py:__init__():42] System info init done +2024-03-12 14:28:55,654 INFO HandlerThread:452226 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:28:55,654 INFO SystemMonitor:452226 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:28:55,654 INFO HandlerThread:452226 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:28:55,655 INFO SystemMonitor:452226 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:28:55,655 INFO SystemMonitor:452226 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:28:55,726 DEBUG HandlerThread:452226 [system_info.py:probe():151] Probing system +2024-03-12 14:28:55,731 DEBUG HandlerThread:452226 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_info.py:probe():199] Probing system done +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:28:55.726177', 'startedAt': '2024-03-12T14:28:55.018301', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_32', '--run_id', 'qformer2_32', '--model.arch_specifier', 'qformer2_32'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-590', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2571.3575529411764, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.75, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.597, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.662, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.777, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.983, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.315, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.081, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.977, 'min': 1500.0, 'max': 2550.0}, {'current': 3006.851, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 52.20142364501953}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942710876465}} +2024-03-12 14:28:55,751 INFO HandlerThread:452226 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:28:55,751 INFO HandlerThread:452226 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:28:55,751 DEBUG HandlerThread:452226 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:28:56,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 14:29:00,119 DEBUG HandlerThread:452226 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:29:00,126 INFO HandlerThread:452226 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:29:00,131 DEBUG SenderThread:452226 [sender.py:send():382] send: files +2024-03-12 14:29:00,131 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:29:00,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:00,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:29:00,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:29:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:00,150 DEBUG SenderThread:452226 [sender.py:send():382] send: telemetry +2024-03-12 14:29:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json +2024-03-12 14:29:01,144 INFO wandb-upload_0:452226 [upload_job.py:push():131] Uploaded file /tmp/tmpsj_hfqi9wandb/lzgq4p2i-wandb-metadata.json +2024-03-12 14:29:02,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:05,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:05,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:10,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:15,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:16,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:21,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:25,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:26,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:27,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:27,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 14:29:30,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:30,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:32,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:32,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:32,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:32,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:32,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:32,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:33,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:35,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:35,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:35,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:35,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:36,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:37,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:37,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:39,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:39,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:39,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:39,158 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:39,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:41,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:43,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:43,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:43,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:43,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:43,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:43,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:45,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:45,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:45,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:47,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:47,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:47,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:47,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:47,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:48,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:49,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:51,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:51,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:51,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:51,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:51,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:53,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:54,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:55,656 DEBUG SystemMonitor:452226 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:29:55,659 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:29:56,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:56,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:56,558 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:56,563 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:56,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:57,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:59,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:00,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:00,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:00,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:00,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:00,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:01,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:04,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:07,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:07,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:07,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:07,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:07,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:07,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:10,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:10,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:10,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:11,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:11,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:14,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:14,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:14,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:14,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:14,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:15,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:15,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:18,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:18,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:18,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:18,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:18,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:19,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:21,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:21,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:21,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:21,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:21,623 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:21,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:23,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:25,661 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:30:26,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:26,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:26,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:26,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:26,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:26,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:27,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:30,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:31,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:31,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:31,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:31,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:31,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:31,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:33,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:35,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:35,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:35,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:35,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:35,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:37,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:39,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:39,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:39,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:39,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:40,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:41,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:42,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:43,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:43,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:43,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:43,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:45,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:45,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:46,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:46,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:46,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:46,480 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:46,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:47,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:48,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:51,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:51,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:51,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:51,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:51,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:53,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:54,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:55,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:55,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:55,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:55,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:55,664 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:30:55,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:57,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:59,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:03,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:03,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:03,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:03,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:03,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:04,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:06,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:06,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:06,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:06,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:07,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:07,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:09,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:10,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:10,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:10,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:11,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:11,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:15,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:19,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:19,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:19,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:19,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:19,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:20,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:21,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:22,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:22,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:22,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:22,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:22,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:23,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:25,666 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:31:25,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:25,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:25,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:25,784 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:25,789 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:26,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:27,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:30,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:30,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:30,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:30,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:30,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:31,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:31,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:35,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:35,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:35,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:35,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:35,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:37,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:37,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:39,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:39,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:39,489 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:39,494 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:39,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:41,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:42,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:45,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:45,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:45,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:45,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:45,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:45,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:47,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:48,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:49,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:49,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:49,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:49,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:49,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:51,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:53,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:53,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:53,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:53,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:53,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:54,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:55,668 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:31:55,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:57,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:57,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:57,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:57,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:57,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:59,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:59,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:02,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:02,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:02,714 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:02,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:02,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:03,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:04,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:09,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:09,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:09,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:09,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:09,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:10,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:11,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:12,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:12,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:12,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:12,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:13,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:13,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:16,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:16,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:16,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:17,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:21,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:21,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:21,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:21,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:21,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:21,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:21,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:24,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:24,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:24,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:24,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:24,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:25,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:32:25,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:26,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:27,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:27,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:27,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:27,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:27,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:29,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:32,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:32,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:32,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:32,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:32,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:32,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:33,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:36,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:36,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:36,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:36,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:36,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:37,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:42,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:44,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:44,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:44,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:44,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:45,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:45,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:48,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:53,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:53,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:53,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:53,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:53,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:53,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:55,677 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:32:55,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:58,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:58,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:58,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:58,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:58,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:59,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:59,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:00,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:03,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:03,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:03,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:03,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:03,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:04,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:05,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:07,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:07,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:07,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:07,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:10,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:10,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:10,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:10,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:10,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:10,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:11,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:14,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:14,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:14,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:14,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:14,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:15,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:15,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:18,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:18,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:18,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:18,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:19,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:19,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:21,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:24,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:24,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:24,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:24,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:24,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:25,674 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:33:25,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:26,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:29,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:29,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:29,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:29,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:29,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:29,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:32,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:34,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:34,258 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:34,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:34,263 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:34,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:35,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:38,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:39,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:39,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:39,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:39,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:39,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:43,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:43,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:43,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:43,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:43,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:48,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:49,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:49,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:49,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:49,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:49,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:51,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:53,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:55,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:55,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:55,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:55,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:55,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:33:55,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:55,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:58,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:58,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:58,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:58,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:59,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:59,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:00,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:02,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:02,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:02,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:02,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:02,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:03,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:05,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:05,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:05,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:06,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:06,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:07,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:09,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:09,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:09,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:09,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:09,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:10,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:11,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:16,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:16,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:16,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:16,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:16,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:16,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:17,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:21,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:21,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:21,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:21,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:21,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:21,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:22,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:25,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:25,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:25,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:25,678 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:34:25,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:25,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:27,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:28,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:28,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:28,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:28,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:28,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:29,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:33,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:33,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:33,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:33,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:33,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:33,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:33,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:36,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:36,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:36,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:36,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:36,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:37,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:38,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:39,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:39,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:39,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:39,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:39,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:41,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:43,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:43,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:43,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:43,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:43,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:43,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:46,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:46,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:46,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:46,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:46,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:47,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:48,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:49,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:49,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:49,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:49,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:49,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:51,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:53,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:53,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:53,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:53,434 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:53,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:53,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:54,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:55,685 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:34:57,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:57,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:57,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:57,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:57,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:59,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:59,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:01,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:01,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:01,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:02,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:02,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:03,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:05,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:05,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:05,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:05,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:08,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:08,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:08,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:08,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:08,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:09,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:10,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:11,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:11,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:11,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:11,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:11,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:13,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:16,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:16,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:16,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:16,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:16,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:17,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:20,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:21,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:21,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:21,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:21,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:21,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:21,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:24,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:24,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:24,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:25,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:35:25,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:26,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:30,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:30,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:30,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:30,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:30,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:31,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:32,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:34,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:34,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:34,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:34,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:35,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:36,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:37,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:38,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:38,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:38,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:38,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:39,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:40,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:43,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:47,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:47,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:47,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:47,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:48,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:48,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:50,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:51,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:51,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:51,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:51,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:52,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:52,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:54,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:55,685 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:35:56,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:56,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:56,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:56,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:57,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:58,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:59,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:59,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:59,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:59,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:59,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:00,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:02,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:05,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:05,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:05,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:06,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:06,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:10,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:10,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:11,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:12,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:13,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:13,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:13,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:13,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:14,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:16,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:16,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:17,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:17,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:17,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:17,948 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:18,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:20,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:21,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:21,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:21,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:21,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:21,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:22,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:22,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:24,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:24,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:25,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:25,687 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:36:26,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:27,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:29,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:29,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:29,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:29,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:30,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:30,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:33,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:33,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:33,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:36,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:37,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:37,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:37,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:38,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:39,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:44,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:45,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:45,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:45,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:45,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:46,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:46,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:49,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:49,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:49,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:49,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:49,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:50,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:50,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:54,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:54,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:54,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:54,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:55,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:55,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:55,689 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:36:56,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:57,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:57,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:57,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:57,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:58,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:58,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:00,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:05,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:05,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:05,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:06,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:06,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:08,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:08,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:08,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:08,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:09,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:10,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:10,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:11,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:11,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:11,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:11,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:12,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:14,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:15,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:15,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:15,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:16,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:16,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:18,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:18,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:18,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:18,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:19,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:20,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:21,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:22,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:22,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:22,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:22,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:23,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:24,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:25,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:25,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:25,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:25,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:25,697 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:37:26,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:26,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:28,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:28,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:28,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:28,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:28,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:29,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:30,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:30,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:30,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:32,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:34,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:34,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:34,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:34,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:35,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:36,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:38,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:38,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:38,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:38,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:38,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:39,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:40,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:43,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:43,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:43,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:43,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:43,843 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:44,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:45,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:46,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:49,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:53,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:53,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:53,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:53,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:54,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:54,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:55,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:55,694 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:37:56,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:56,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:56,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:56,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:57,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:58,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:00,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:00,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:00,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:02,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:02,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:02,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:02,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:03,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:04,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:05,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:06,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:06,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:06,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:06,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:07,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:08,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:09,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:09,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:09,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:09,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:10,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:10,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:12,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:13,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:13,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:13,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:13,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:14,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:14,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:15,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:16,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:17,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:17,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:17,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:17,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:18,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:20,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:21,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:24,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:24,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:24,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:24,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:25,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:25,702 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:38:26,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:27,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:27,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:27,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:27,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:27,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:28,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:28,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:30,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:30,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:30,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:30,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:30,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:31,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:32,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:33,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:34,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:34,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:34,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:35,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:36,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:38,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:38,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:38,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:38,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:38,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:39,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:40,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:43,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:44,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:44,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:44,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:44,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:45,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:45,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:46,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:47,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:47,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:47,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:48,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:48,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:50,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:51,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:51,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:51,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:51,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:52,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:52,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:54,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:55,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:55,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:55,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:55,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:55,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:55,699 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:38:56,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:59,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:59,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:59,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:59,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:59,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:00,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:00,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:00,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:00,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:02,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:02,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:02,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:02,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:03,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:04,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:04,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:06,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:06,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:06,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:06,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:06,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:08,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:10,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:11,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:11,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:11,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:11,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:12,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:14,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:15,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:15,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:15,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:15,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:15,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:19,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,373 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:19,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:21,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:22,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:22,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:22,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:22,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:23,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:24,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:25,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:39:26,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:26,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:26,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:26,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:26,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:27,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:28,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:29,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:29,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:29,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:29,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:30,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:30,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:30,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:32,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:33,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:33,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:33,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:33,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:34,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:34,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:36,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:36,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:36,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:36,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:37,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:37,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:38,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:39,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:39,472 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:39,472 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:39,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:40,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:40,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:43,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:45,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:45,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:45,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:48,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:48,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:48,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:48,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:48,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:49,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:50,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:54,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:54,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:54,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:54,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:55,703 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:39:56,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:59,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:00,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:00,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:03,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:03,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:03,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:03,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:04,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:04,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:04,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:09,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:12,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:12,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:14,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:15,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:15,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:15,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:15,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:15,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:15,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:15,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:16,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:16,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:19,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:19,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:19,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:19,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:20,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:20,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:23,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:23,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:23,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:23,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:23,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:24,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:25,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:40:25,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:26,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:26,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:26,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:26,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:27,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:28,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:30,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:31,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:31,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:31,098 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:31,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:31,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:31,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:32,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:35,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:35,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:35,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:35,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:35,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:36,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:36,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:39,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:39,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:39,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:39,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:39,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:40,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:42,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:44,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:44,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:44,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:44,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:44,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:45,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:46,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:47,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:47,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:47,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:47,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:47,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:48,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:48,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:51,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:51,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:51,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:51,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:52,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:52,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:54,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:55,708 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:40:58,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:00,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:00,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:00,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:00,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:01,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:02,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:03,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:03,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:03,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:03,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:03,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:04,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:06,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:07,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:07,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:07,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:07,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:08,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:08,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:09,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:10,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:10,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:10,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:11,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:12,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:14,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:15,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:15,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:15,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:15,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:16,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:19,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:19,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:19,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:19,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:19,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:20,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:20,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:22,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:22,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:22,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:22,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:23,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:24,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:25,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:25,710 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:41:26,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:26,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:26,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:26,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:28,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:30,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:31,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:31,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:31,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:31,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:31,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:32,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:32,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:35,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:35,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:35,558 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:35,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:36,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:36,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:36,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:39,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:39,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:39,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:39,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:39,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:40,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:42,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:42,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:42,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:42,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:42,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:43,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:44,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:45,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:46,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:46,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:46,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:46,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:46,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:47,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:48,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:49,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:49,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:49,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:49,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:50,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:50,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:52,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:55,718 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:41:57,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:57,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:57,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:57,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:57,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:02,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:02,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:02,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:02,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:02,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:03,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:04,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:07,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:07,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:07,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:07,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:08,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:09,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:10,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:10,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:10,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:10,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:11,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:12,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:14,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:14,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:14,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:14,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:14,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:15,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:18,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:18,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:18,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:18,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:20,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:20,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:25,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:25,714 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:42:26,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:26,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:26,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:26,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:26,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:28,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:30,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:30,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:30,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:30,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:31,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:32,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:34,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:34,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:34,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:34,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:35,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:36,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:38,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:38,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:38,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:38,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:39,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:40,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:40,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:44,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:44,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:44,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:44,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:44,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:46,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:47,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:47,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:47,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:47,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:47,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:48,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:51,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:52,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:52,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:52,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:52,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:52,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:54,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:42:56,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:56,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:56,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:56,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:56,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:57,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:58,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:59,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:59,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:59,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:59,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:00,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:00,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:00,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:02,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:04,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:04,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:04,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:04,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:04,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:06,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:08,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:09,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:09,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:09,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:09,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:09,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:10,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:12,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:12,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:12,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:12,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:13,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:13,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:14,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:15,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:16,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:16,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:16,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:16,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:16,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:18,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:19,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:21,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:21,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:21,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:21,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:21,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:22,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:25,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:25,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:25,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:25,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:25,718 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:43:25,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:26,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:29,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:29,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:29,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:29,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:29,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:30,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:30,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:34,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:34,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:34,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:34,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:34,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:35,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:36,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:37,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:37,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:37,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:37,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:38,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:38,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:40,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:41,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:42,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:45,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:45,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:45,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:45,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:45,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:46,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:46,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:50,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:50,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,051 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:50,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:51,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:54,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:54,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:43:56,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:56,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:57,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:57,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:57,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:57,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:57,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:58,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:00,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:00,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:00,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:01,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:01,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:01,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:01,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:01,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:01,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:02,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:05,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:05,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:05,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:05,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:05,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:06,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:07,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:09,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:09,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:09,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:10,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:12,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:13,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:13,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:13,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:13,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:13,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:14,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:15,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:17,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:17,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:17,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:17,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:17,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:18,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:18,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:20,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:20,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:20,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:20,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:20,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:22,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:23,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:25,723 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:44:28,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:29,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:29,841 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:29,841 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:29,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:29,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:30,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:34,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:34,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:36,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:39,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:40,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:40,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:40,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:40,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:40,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:42,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:43,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:43,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:43,794 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:43,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:43,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:44,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:45,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:47,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:47,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:47,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:47,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:47,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:48,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:50,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,725 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:44:55,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:55,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:55,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:55,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:55,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:56,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:00,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:00,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:00,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:00,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:00,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:00,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:01,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:04,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:04,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:04,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:04,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:04,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:06,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:06,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:08,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:08,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:08,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:08,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:08,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:10,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:12,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:14,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:14,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:14,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:14,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:14,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:16,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:17,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:17,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:17,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:17,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:17,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:18,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:22,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:23,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:23,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:23,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:23,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:23,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:24,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:25,727 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:45:26,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:26,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:26,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:26,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:26,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:28,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:30,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:30,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:30,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:30,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:30,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:32,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:33,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:33,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:33,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:33,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:33,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:33,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:35,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:37,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:37,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:37,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:37,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:38,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:39,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:39,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:41,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:41,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:42,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:43,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:44,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:45,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:45,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:45,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:45,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:46,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:47,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:49,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:49,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:49,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:49,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:50,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:50,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:51,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:55,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:45:57,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:57,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:57,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:57,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:58,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:59,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:00,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:00,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:01,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:01,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:01,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:01,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:02,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:03,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:05,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:05,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:05,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:05,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:06,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:06,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:07,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:08,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:08,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:08,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:08,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:09,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:11,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:11,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:11,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:11,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:11,629 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:11,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:12,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:13,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:15,592 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:15,592 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:15,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:16,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:16,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:17,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:18,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:18,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:18,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:18,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:19,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:21,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:21,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:22,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:22,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:22,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:22,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:23,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:25,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:25,732 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:46:26,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:26,279 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:26,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:26,284 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:27,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:27,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:29,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:32,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:34,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:34,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:34,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:34,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:35,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:37,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:37,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:40,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:40,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:40,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:40,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:41,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:43,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:43,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:49,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:49,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:49,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:49,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:49,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:50,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:51,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:52,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:52,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:54,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:55,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:55,734 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:00,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:00,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:00,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:00,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:01,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:03,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:04,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:04,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:04,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:04,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:05,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:05,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:05,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:07,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:07,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:07,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:07,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:09,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:10,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:11,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:11,221 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:11,221 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:11,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:12,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:13,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:14,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:14,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:14,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:15,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:15,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:16,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:17,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:18,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:18,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:18,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:18,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:19,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:21,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:22,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:25,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:25,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:25,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:25,110 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:25,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:25,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:27,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:27,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:28,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:28,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:28,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:28,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:29,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:31,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:31,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:31,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:31,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:31,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:32,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:33,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:33,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:38,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:38,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:38,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:38,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:39,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:39,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:39,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:41,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:41,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:41,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:41,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:43,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:44,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:46,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:46,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:47,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:49,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:50,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:50,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:50,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:50,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:50,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:51,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:53,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:53,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:53,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:53,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:53,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:54,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:55,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:55,739 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:55,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:59,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:59,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:59,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:59,078 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:59,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:00,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:01,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:01,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:02,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:02,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:02,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:02,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:03,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:05,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:05,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:05,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:05,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:05,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:06,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:06,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:07,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:09,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:09,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:09,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:09,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:10,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:11,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:12,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:15,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:17,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:17,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:17,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:18,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:19,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:21,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:21,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:21,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:21,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:22,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:22,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:23,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:24,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:24,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:24,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:24,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:25,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:25,743 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:48:27,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:27,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:29,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:29,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:29,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:29,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:30,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:30,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:30,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:31,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:32,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:32,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:32,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:32,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:32,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:33,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:35,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:37,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:38,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:38,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:38,898 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:38,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:39,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:41,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:42,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:42,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:42,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:42,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:43,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:43,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:43,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:47,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:47,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:47,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:47,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:48,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:49,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:49,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:51,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:51,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:51,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:51,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:51,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:53,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:55,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:55,743 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:48:56,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:56,203 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:56,203 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:56,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:56,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:57,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:59,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:59,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:59,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:59,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:00,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:01,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:04,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:04,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:04,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:04,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:05,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:05,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:05,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:07,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:07,735 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:07,735 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:07,740 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:08,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:09,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:10,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:11,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:11,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:11,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:11,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:11,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:13,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:14,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:14,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:14,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:14,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:15,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:15,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:17,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:19,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:19,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:19,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:19,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:19,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:21,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:22,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:24,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:24,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:24,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:24,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:25,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:25,745 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:49:27,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:27,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:28,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:28,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:28,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:28,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:29,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:30,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:31,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:32,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:32,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:32,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:32,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:33,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:33,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:33,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:36,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:36,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:36,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:36,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:37,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:37,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:38,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:40,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:40,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:40,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:40,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:40,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:41,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:44,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:47,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:47,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:47,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:47,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:48,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:49,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:49,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:51,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:51,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:51,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:51,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:52,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:53,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:55,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:55,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:55,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:55,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:55,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:55,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:55,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:49:57,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:59,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:59,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:59,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:59,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:00,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:00,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:01,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:03,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:03,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:03,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:03,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:04,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:05,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:05,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:06,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:06,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:06,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:06,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:07,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:09,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:10,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:10,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:10,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:10,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:11,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:11,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:11,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:14,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:14,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:14,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:14,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:14,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:15,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:17,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:17,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:17,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:17,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:17,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:18,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:19,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:22,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:23,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:23,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:23,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:23,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:23,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:25,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:25,749 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:50:27,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:28,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:28,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:28,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:28,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:29,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:31,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:32,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:32,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:32,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:32,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:32,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:33,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:35,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:37,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:38,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:38,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:38,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:38,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:38,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:39,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:42,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:42,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:42,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:42,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:43,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:43,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:45,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:47,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:47,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:47,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:47,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:48,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:49,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:49,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:50,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:50,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:50,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:50,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:51,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:53,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:54,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:54,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:54,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:54,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:55,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:55,752 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:50:57,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:57,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:57,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:57,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:58,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:59,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:01,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:02,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:02,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:02,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:02,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:03,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:06,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:06,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:06,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:06,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:06,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:10,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:10,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:10,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:10,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:11,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:11,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:13,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:14,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:14,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:14,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:14,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:15,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:17,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:17,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:21,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:21,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:21,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:21,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:21,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:23,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:23,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:25,754 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:51:28,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:30,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:30,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:30,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:30,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:31,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:31,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:34,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:38,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:38,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:38,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:38,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:39,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:39,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:39,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:44,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:47,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:47,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:47,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:47,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:47,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:49,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:50,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:52,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:52,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:52,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:52,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:52,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:53,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:55,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:55,759 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:00,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:00,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:00,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:00,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:00,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:01,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:03,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:04,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:04,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:04,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:04,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:04,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:05,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:06,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:07,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:07,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:07,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:07,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:08,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:09,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:11,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:11,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:11,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:11,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:11,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:12,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:13,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:16,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:16,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:16,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:16,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:16,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:17,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:19,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:21,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:21,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:21,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:21,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:21,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:22,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:23,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:25,759 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:27,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:27,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:27,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:27,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:27,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:27,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:29,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:31,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:31,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:31,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:31,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:31,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:33,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:33,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:36,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:36,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:36,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:36,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:36,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:37,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:38,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:40,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:40,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:40,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:40,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:41,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:43,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:43,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:45,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:45,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:45,356 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:45,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:47,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:48,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:48,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:48,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:48,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:48,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:49,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:49,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:52,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:52,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:52,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:52,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:53,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:53,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:54,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:55,764 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:59,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:00,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:00,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:00,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:00,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:01,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:03,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:04,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:05,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:05,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:05,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:05,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:05,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:07,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:08,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:08,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:08,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:08,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:09,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:09,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:11,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:14,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:14,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:14,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:14,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:14,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:15,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:15,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:18,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:18,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:18,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:18,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:18,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:19,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:20,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:25,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:25,763 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:53:26,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:26,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:26,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:26,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:27,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:27,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:30,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:30,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:30,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:30,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:30,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:31,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:31,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:33,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:33,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:34,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:34,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:35,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:36,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:38,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:38,159 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:38,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:38,164 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:38,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:39,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:41,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:41,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:41,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:41,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:41,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:42,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:43,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:45,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:45,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:45,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:45,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:45,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:46,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:47,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:48,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:48,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:48,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:48,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:49,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:49,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:51,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:53,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:53,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:53,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:53,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:53,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:55,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:55,765 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:53:56,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:59,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:59,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:59,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:59,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:59,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:01,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:02,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:03,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:03,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:03,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:03,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:04,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:05,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:07,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:08,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:08,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:08,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:08,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:08,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:09,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:13,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:13,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:13,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:13,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:13,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:14,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:15,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:17,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:17,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:17,845 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:17,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:18,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:18,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:21,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:21,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:21,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:21,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:21,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:23,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:24,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:25,768 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:54:26,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:26,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:26,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:26,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:26,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:27,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:29,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:29,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:29,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:29,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:29,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:30,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:31,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:33,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:33,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:33,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:33,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:33,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:35,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:35,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:37,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:37,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:37,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:37,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:37,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:39,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:40,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:42,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:42,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:42,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:42,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:43,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:46,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:46,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:46,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:46,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:46,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:47,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:47,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:50,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:50,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:50,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:50,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:51,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:51,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:51,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:54,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:54,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:54,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:54,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:54,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:55,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:55,769 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:54:56,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:57,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:57,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:57,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:57,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:57,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:59,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:00,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:00,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:00,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:01,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:01,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:02,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:04,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:04,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:04,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:04,258 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:04,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:05,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:08,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:09,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:09,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:09,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:09,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:09,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:11,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:13,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:13,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:13,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:13,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:13,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:14,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:15,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:18,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:18,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:18,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:18,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:19,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:20,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:21,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:24,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:24,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:24,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:24,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:24,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:25,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:25,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:25,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:55:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:30,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:30,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:30,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:30,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:30,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:31,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:35,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:35,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:35,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:35,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:35,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:35,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:37,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:40,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:40,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:40,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:40,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:40,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:41,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:41,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:44,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:44,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:44,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:44,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:44,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:45,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:46,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:48,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:48,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:48,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:48,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:48,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:49,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:51,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:53,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:53,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:53,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:53,029 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:53,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:55,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:55,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:55:56,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:57,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:57,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:57,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:57,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:57,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:59,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:02,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:03,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:03,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:03,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:03,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:03,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:05,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:08,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:11,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:11,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:11,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:11,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:11,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:13,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:13,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:15,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:15,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:15,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:15,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:15,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:17,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:18,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:18,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:18,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:18,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:18,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:19,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:19,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:22,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:22,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:22,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:22,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:22,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:23,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:24,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:25,782 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:56:26,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:26,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:26,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:26,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:27,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:27,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:30,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:30,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:30,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:30,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:30,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:30,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:31,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:33,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:33,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:33,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:33,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:34,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:35,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:35,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:37,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:37,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:37,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:37,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:37,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:39,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:41,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:41,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:42,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:43,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:45,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:45,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:45,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:45,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:45,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:46,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:47,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:49,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:49,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:49,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:49,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:49,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:51,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:52,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:52,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:52,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:52,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:52,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:53,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:53,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:55,778 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:56:56,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:56,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:56,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:56,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:56,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:57,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:57,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:00,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:00,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:00,348 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:00,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:00,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:01,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:03,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:05,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:05,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:05,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:05,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:05,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:07,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:08,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:13,898 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:13,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:13,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:14,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:15,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:15,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:19,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:22,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:22,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:22,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:22,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:23,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:23,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:24,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:25,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:57:26,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:26,137 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:26,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:26,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:26,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:27,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:30,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:30,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:30,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:30,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:30,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:30,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:31,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:33,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:33,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:33,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:33,798 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:34,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:35,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:35,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:37,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:37,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:37,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:37,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:37,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:39,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:41,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:42,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:42,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:42,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:42,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:42,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:43,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:46,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:46,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:46,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:46,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:46,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:46,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:47,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:51,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:51,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:51,077 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:51,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:51,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:52,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:53,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:55,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:57:57,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:00,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:00,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:00,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:00,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:00,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:01,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:03,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:08,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:08,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:08,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:08,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:09,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:13,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:16,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:16,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:16,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:16,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:17,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:18,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:19,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:19,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:19,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:19,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:20,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:21,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:23,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:24,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:24,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:24,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:24,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:24,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:25,785 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:58:25,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:28,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:28,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:28,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:28,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:29,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:29,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:31,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:31,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:31,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:31,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:31,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:33,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:34,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:34,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:34,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:34,578 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:34,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:34,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:35,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:37,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:37,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:37,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:37,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:37,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:39,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:39,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:41,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:41,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:41,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:41,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:41,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:41,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:45,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:45,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:45,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:45,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:45,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:45,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:47,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:50,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:51,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:51,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:51,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:51,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:51,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:51,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:55,787 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:58:55,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:57,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:57,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:57,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:57,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:59,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:00,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:00,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:00,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:00,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:00,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:01,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:01,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:04,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:04,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:04,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:04,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:04,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:05,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:07,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:10,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:10,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:10,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:10,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:10,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:11,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:13,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:13,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:13,245 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:13,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:13,250 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:13,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:15,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:17,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:17,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:17,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:17,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:17,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:18,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:19,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:20,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:20,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:20,809 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:20,814 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:20,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:21,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:23,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:25,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:25,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:25,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:25,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:25,790 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:59:25,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:27,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:29,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:30,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:30,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:30,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:30,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:30,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:31,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:34,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:34,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:34,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:34,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:34,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:35,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:35,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:37,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:37,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:37,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:37,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:38,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:39,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:40,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:43,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:43,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:43,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:43,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:43,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:43,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:46,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:51,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:51,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:51,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:51,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:51,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:51,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:53,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:55,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:59:56,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:59,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:59,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:59,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:59,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:59,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:01,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:02,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:03,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:03,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:03,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:03,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:03,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:05,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:07,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:07,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:07,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:07,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:07,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:07,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:08,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:10,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:10,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:10,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:10,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:10,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:11,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:13,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:14,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:14,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:14,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:14,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:14,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:15,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:15,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:18,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:23,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:23,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:23,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:23,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:25,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:00:25,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:26,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:26,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:26,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:26,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:26,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:27,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:28,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:30,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:30,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:30,395 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:30,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:30,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:31,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:33,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:33,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:33,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:33,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:33,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:35,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:37,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:37,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:37,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:37,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:37,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:39,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:39,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:41,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:41,294 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:41,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:41,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:41,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:43,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:44,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:44,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:44,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:44,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:44,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:44,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:45,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:50,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,237 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:50,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:50,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:51,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:55,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:55,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:00:58,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:58,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:58,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:58,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:59,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:00,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:02,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:02,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:02,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:02,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:03,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:06,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:07,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:07,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:07,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:07,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:07,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:07,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:10,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:10,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:10,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:10,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:11,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:11,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:14,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:14,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:14,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:14,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:14,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:15,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:17,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:22,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:23,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:23,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:23,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:23,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:25,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:01:25,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:27,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:27,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:27,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:27,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:27,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:27,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:28,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:31,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:31,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:31,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:31,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:31,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:33,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:33,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:34,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:34,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:34,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:34,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:34,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:35,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:38,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:38,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:38,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:38,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:38,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:39,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:43,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:43,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:43,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:43,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:43,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:44,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:45,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:45,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:47,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:47,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:47,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:49,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:52,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:52,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:52,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:52,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:53,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:54,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:55,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:55,801 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:01:55,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:55,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:55,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:55,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:56,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:58,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:00,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:00,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:00,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:01,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:01,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:02,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:03,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:03,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:03,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:03,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:04,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:06,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:07,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:07,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:07,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:08,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:08,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:10,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:10,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:10,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:10,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:11,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:11,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:14,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:14,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:14,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:14,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:15,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:16,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:17,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:19,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:19,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:19,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:19,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:20,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:22,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:22,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:23,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:23,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:23,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:23,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:24,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:24,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:25,803 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:02:27,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:28,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:28,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:28,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:28,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:29,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:30,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:33,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:37,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:37,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:37,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:37,115 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:38,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:38,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:39,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:42,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:42,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:42,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:42,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:44,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:44,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:45,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:46,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:46,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:46,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:46,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:48,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:50,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:51,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:51,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:51,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:52,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:55,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:55,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:55,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:55,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:55,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:55,805 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:02:56,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:56,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:59,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:59,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:59,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:59,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:00,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:00,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:00,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:00,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:03,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:03,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:03,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:03,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:04,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:04,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:06,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:07,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:07,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:07,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:07,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:08,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:11,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:11,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:11,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:11,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:11,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:12,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:12,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:15,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:15,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:15,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:15,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:16,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:17,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:18,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:18,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:18,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:18,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:18,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:19,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:20,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:21,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:21,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:21,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:21,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:22,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:22,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:25,807 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:03:27,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:27,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:27,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:27,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:28,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:28,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:28,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:30,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:30,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:30,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:30,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:31,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:32,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:33,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:38,562 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:38,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:38,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:39,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:39,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:40,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:42,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:42,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:42,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:42,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:43,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:44,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:46,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:46,288 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:46,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:46,293 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:47,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:48,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:50,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:50,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:50,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:50,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:50,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:51,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:52,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:55,809 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:03:56,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:56,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:56,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:56,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:57,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:58,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:00,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:00,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:00,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:00,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:01,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:01,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:02,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:03,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:03,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:03,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:03,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:04,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:06,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:06,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:07,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:07,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:07,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:07,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:08,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:10,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:11,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:16,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:16,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:16,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:16,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:17,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:18,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:20,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:20,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:20,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:20,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:22,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:22,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:25,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:25,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:25,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:25,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:25,815 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:04:26,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:27,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:28,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:30,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:30,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:30,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:31,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:32,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:33,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:36,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:36,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:36,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:36,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:37,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:38,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:39,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:44,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:44,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:44,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:44,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:44,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:45,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:46,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:48,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:48,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:48,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:48,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:48,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:50,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:50,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:53,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:53,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:53,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:53,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:54,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:54,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:55,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:55,814 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:04:56,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:56,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:56,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:56,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:57,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:58,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:00,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:00,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:00,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:00,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:00,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:00,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:00,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:02,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:04,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:04,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:04,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:04,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:05,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:05,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:06,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:08,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:08,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:08,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:08,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:09,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:10,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:11,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:12,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:12,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:12,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:12,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:13,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:14,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:16,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:18,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:18,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:18,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:18,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:19,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:20,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:21,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:21,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:21,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:21,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:21,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:22,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:22,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:24,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:24,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:24,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:24,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:25,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:25,826 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:05:26,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:27,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:28,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:28,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:28,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:28,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:29,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:30,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:30,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:32,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:32,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:32,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:32,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:32,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:33,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:34,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:38,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:38,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:38,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:38,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:38,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:40,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:43,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:44,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:44,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:44,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:44,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:44,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:45,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:46,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:48,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:48,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:48,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:48,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:48,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:49,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:50,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:53,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:55,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:55,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:55,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:55,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:55,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:55,818 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:05:56,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:59,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:59,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:59,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:59,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:59,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:00,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:00,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:02,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:03,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:03,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:03,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:03,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:04,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:05,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:06,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:06,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:06,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:06,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:06,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:07,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:08,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:10,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:10,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:10,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:10,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:10,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:11,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:12,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:16,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:16,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:16,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:16,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:16,441 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:17,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:18,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:21,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:23,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:23,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:23,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:23,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:24,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:24,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:25,824 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:06:26,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:27,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:27,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:27,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:27,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:28,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:30,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:31,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:31,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:31,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:32,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:34,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:36,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:37,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:37,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:37,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:37,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:38,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:38,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:41,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:41,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:41,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:41,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:42,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:42,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:42,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:46,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:46,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:46,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:46,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:47,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:48,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:48,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:50,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:50,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:50,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:50,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:51,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:52,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:53,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:54,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:54,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:54,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:54,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:54,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:55,823 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:06:56,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:58,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:58,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:58,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:58,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:58,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:59,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:00,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:03,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:04,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:04,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:08,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:08,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:08,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:08,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:09,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:09,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:10,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:14,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:14,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:14,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:14,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:14,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:16,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:19,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:19,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:19,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:19,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:20,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:20,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:20,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:23,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:23,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:23,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:23,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:24,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:24,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:25,825 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:07:25,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:26,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:26,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:26,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:26,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:27,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:28,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:31,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:32,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:32,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:32,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:32,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:32,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:34,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:36,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:36,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:36,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:36,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:36,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:37,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:38,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:40,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:40,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:40,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:40,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:40,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:42,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:42,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:45,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:45,001 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:45,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:45,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:45,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:46,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:47,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:48,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:48,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:48,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:48,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:49,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:50,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:52,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:52,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:52,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:52,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:52,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:53,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:54,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:55,827 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:07:56,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:56,734 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:56,735 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:56,740 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:57,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:57,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:58,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:01,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:01,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:01,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:01,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:01,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:02,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:03,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:07,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:07,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:07,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:07,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:07,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:08,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:08,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:13,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:15,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:15,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:15,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:15,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:15,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:16,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:18,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:18,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:20,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:20,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:20,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:20,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:21,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:22,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:24,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:24,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:24,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:24,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:24,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:24,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:25,829 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:08:26,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:27,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:27,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:27,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:27,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:29,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:30,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:32,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:32,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:32,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:32,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:33,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:34,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:34,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:37,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:37,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:37,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:37,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:37,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:38,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:40,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:41,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:41,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:41,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:41,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:42,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:44,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:45,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:45,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:45,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:46,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:46,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:49,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:49,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:49,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:49,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:50,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:50,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:52,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:52,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:52,890 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:52,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:52,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:53,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:54,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:55,834 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:08:55,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:56,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:56,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:56,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:56,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:56,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:58,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:01,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:01,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:01,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:01,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:01,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:01,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:02,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:04,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:04,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:04,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:04,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:06,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:06,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:07,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:07,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:07,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:07,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:07,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:08,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:11,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:11,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:11,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:11,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:11,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:12,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:12,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:15,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:15,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:15,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:15,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:16,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:17,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:18,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:18,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:18,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:18,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:18,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:20,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:21,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:21,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:21,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:21,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:22,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:22,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:24,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:25,834 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:09:26,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:26,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:26,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:26,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:26,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:28,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:28,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:30,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:30,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:30,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:31,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:31,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:32,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:34,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:35,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:35,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:35,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:35,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:36,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:36,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:39,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:42,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:42,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:42,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:42,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:42,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:44,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:46,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:46,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:46,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:46,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:46,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:48,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:50,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:50,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:50,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:50,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:50,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:50,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:52,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:53,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:53,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:53,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:53,777 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:54,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:54,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:55,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:55,839 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:09:57,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:57,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:57,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:57,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:57,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:58,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:00,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:00,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:00,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:00,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:00,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:01,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:02,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:04,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:04,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:04,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:04,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:04,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:06,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:06,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:08,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:08,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:08,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:08,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:08,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:10,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:11,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:11,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:11,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:11,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:11,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:12,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:14,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:15,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:15,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:15,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:15,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:15,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:16,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:17,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:18,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:18,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:18,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:18,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:19,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:20,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:22,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:23,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:23,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:23,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:23,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:23,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:24,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:25,838 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:10:27,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:28,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:28,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:28,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:28,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:28,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:30,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:30,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:33,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:33,071 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:33,071 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:33,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:33,076 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:33,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:34,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:35,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:35,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:35,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:36,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:36,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:38,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:39,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:39,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:39,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:39,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:39,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:40,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:43,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:43,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:43,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:43,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:44,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:44,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:46,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:46,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:47,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:48,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:50,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:50,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:50,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:50,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:50,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:50,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:52,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:55,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:55,843 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:10:56,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:56,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:56,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:56,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:57,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:58,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:01,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:01,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:01,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:01,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:01,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:01,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:02,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:05,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:05,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:05,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:05,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:06,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:07,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:12,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:13,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:13,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:13,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:13,830 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:14,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:14,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:15,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:15,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:17,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:17,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:17,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:17,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:17,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:17,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:18,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:21,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:21,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:22,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:23,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:25,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:25,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:25,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:25,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:25,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:25,843 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:11:26,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:28,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:29,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:29,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:29,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:29,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:29,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:30,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:30,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:32,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:32,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:32,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:32,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:32,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:34,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:34,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:37,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:37,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:37,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:37,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:37,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:38,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:40,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:43,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:43,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:43,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:43,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:43,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:44,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:45,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:46,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:46,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:46,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:46,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:47,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:48,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:50,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:51,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:51,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:51,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:51,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:51,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:52,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:55,845 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:11:56,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:59,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:59,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:59,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:59,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:00,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:00,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:02,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:05,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:05,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:05,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:05,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:05,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:06,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:08,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:09,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:09,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:09,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:09,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:09,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:10,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:13,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:14,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:14,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:14,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:14,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:15,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:15,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:16,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:18,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:18,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:18,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:18,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:18,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:19,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:20,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:23,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:23,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:23,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:23,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:23,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:24,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:25,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:25,847 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:12:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:26,294 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:26,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:26,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:26,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:28,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:33,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:33,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:33,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:33,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:33,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:34,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:38,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:38,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:38,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:38,554 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:38,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:40,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:41,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:42,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:42,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:42,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:42,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:42,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:42,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:45,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:45,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:45,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:45,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:45,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:46,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:47,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:52,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:52,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:52,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:52,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:52,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:53,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:54,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:55,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:55,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:55,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:55,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:55,849 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:12:56,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:56,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:58,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:59,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:59,482 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:59,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:59,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:59,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:00,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:00,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:04,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:04,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:04,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:04,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:04,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:04,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:06,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:08,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:08,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:08,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:08,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:08,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:10,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:10,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:12,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:12,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:12,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:12,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:12,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:14,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:15,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:16,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:16,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:16,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:16,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:16,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:18,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:20,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:25,071 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:25,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:25,076 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:25,851 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:13:25,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:26,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:28,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:28,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:28,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:28,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:28,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:30,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:30,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:31,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:34,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:34,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:34,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:34,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:34,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:36,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:36,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:38,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:38,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:38,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:38,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:38,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:40,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:41,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:41,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:41,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:41,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:42,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:42,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:42,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:46,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:46,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:46,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:46,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:46,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:48,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:48,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:49,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:49,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:49,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:49,670 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:49,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:50,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:53,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:55,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:55,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:55,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:55,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:55,853 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:13:55,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:56,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:58,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:59,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:59,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:59,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:59,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:59,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:00,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:00,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:02,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:02,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:02,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:02,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:02,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:04,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:04,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:06,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:06,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:06,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:06,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:06,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:08,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:10,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:10,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:10,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:10,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:10,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:10,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:10,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:15,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:16,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:16,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:16,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:16,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:16,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:18,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:19,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:19,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:19,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:19,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:19,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:20,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:20,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:23,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:23,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:24,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:25,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:25,860 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:14:29,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:29,592 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:29,592 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:29,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:29,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:30,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:30,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:30,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:33,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:33,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:33,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:33,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:33,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:34,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:37,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:37,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:37,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:37,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:37,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:38,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:41,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:41,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:41,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:41,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:41,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:41,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:42,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:45,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:46,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:46,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:46,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:46,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:46,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:46,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:48,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:51,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:51,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:51,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:51,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:51,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:51,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:52,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:55,857 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:14:56,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:58,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:58,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:58,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:58,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:59,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:00,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:02,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:02,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:02,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:02,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:02,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:02,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:04,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:06,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:06,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:06,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:06,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:06,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:07,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:08,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:10,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:10,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:10,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:10,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:10,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:12,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:13,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:13,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:13,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:13,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:13,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:14,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:17,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:17,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:17,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:17,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:17,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:18,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:18,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:23,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:23,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:23,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:23,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:23,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:24,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:24,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:25,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:15:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:26,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:26,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:26,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:26,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:26,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:29,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:30,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:30,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:30,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:30,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:30,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:32,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:34,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:34,246 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:34,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:34,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:34,251 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:34,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:34,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:37,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:37,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:38,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:40,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:42,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:42,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:42,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:42,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:42,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:42,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:45,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:48,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:48,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:48,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:48,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:49,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:51,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:51,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:51,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:51,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:51,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:51,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:52,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:53,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:55,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:15:56,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:57,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:57,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:57,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:58,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:59,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:00,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:02,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:02,137 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:02,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:02,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:02,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:03,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:03,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:06,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:06,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:06,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:06,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:07,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:07,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:09,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:10,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:10,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:10,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:10,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:10,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:11,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:13,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:13,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:13,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:13,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:13,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:14,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:15,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:16,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:16,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:16,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:16,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:17,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:18,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:19,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:21,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:21,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:21,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:21,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:21,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:23,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:24,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:25,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:25,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:25,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:25,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:25,867 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:16:26,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:27,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:29,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:29,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:29,052 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:29,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:29,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:30,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:30,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:31,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:32,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:32,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:32,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:32,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:33,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:34,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:35,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:36,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:36,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:36,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:36,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:36,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:37,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:39,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:39,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:39,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:39,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:39,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:40,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:41,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:44,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:44,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:44,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:44,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:45,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:45,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:47,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:47,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:47,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:47,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:48,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:49,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:50,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:51,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:51,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:51,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:51,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:52,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:53,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:55,866 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:16:55,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:58,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:58,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:58,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:59,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:00,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:01,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:01,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:01,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:01,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:01,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:01,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:02,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:03,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:06,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:06,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:06,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:06,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:06,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:07,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:07,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:09,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:09,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:09,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:09,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:10,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:11,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:12,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:13,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:13,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:13,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:13,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:13,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:15,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:17,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:17,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:17,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:17,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:17,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:18,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:19,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:22,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:25,868 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:17:26,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:26,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:26,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:26,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:27,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:27,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:28,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:29,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:29,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:29,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:29,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:30,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:31,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:33,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:33,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:33,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:33,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:33,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:34,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:35,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:36,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:36,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:36,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:36,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:37,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:38,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:39,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:42,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:42,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:42,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:42,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:44,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:45,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:45,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:47,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:47,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:47,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:47,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:48,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:49,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:49,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:52,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:52,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:52,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:52,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:53,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:54,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:55,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:55,870 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:17:57,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:57,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:57,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:57,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:57,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:59,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:00,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:00,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:02,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:02,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:02,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:02,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:03,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:05,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:05,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:08,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:08,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:08,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:08,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:08,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:09,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:11,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:11,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:11,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:11,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:11,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:12,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:13,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:15,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:16,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:16,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:16,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:16,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:16,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:20,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:20,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:20,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:20,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:20,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:22,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:24,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:24,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:24,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:24,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:25,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:25,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:25,872 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:18:27,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:27,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:27,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:27,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:27,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:28,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:29,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:33,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:34,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:34,690 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:34,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:34,695 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:35,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:37,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:38,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:38,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:38,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:38,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:38,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:41,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:43,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:44,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:44,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:44,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:44,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:45,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:45,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:47,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:48,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:48,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:48,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:48,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:48,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:49,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:51,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:52,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:52,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:53,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:53,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:54,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:55,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:55,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:55,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:55,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:55,874 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:18:56,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:57,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:58,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:58,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:58,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:58,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:59,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:59,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:00,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:01,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:03,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:03,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:03,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:03,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:04,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:04,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:05,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:08,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:08,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:08,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:08,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:09,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:09,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:10,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:11,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:11,872 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:11,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:11,877 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:12,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:13,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:15,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:16,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:16,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:16,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:16,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:16,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:17,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:19,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:20,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:20,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:20,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:20,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:21,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:21,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:23,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:23,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:23,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:23,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:23,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:24,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:25,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:25,876 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:19:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:27,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:27,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:28,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:29,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:31,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:31,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:31,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:31,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:31,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:32,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:33,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:34,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:34,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:34,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:34,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:35,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:37,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:37,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:38,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:38,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:38,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:38,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:38,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:39,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:41,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:41,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:41,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:41,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:42,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:42,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:43,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:46,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:46,498 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:46,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:46,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:47,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:47,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:48,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:51,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:51,193 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:51,193 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:51,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:51,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:53,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:54,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:55,881 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:19:59,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:59,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:59,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:59,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:59,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:00,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:01,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:04,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:04,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:04,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:04,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:04,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:05,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:05,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:08,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:08,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:08,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:08,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:09,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:09,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:10,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:11,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:11,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:11,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:11,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:12,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:13,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:15,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:15,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:15,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:15,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:16,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:17,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:18,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:18,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:18,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:18,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:21,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:22,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:22,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:22,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:22,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:23,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:25,880 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:20:25,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:25,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:25,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:25,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:26,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:26,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:27,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:31,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:31,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:31,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:31,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:32,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:32,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:33,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:36,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:36,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:36,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:36,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:37,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:37,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:39,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:39,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:39,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:39,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:39,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:40,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:41,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:42,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:43,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:43,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:43,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:43,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:44,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:45,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:48,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:48,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:48,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:48,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:48,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:51,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:54,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:55,885 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:20:58,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:58,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:58,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:58,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:59,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:59,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:01,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:02,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:02,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:02,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:02,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:03,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:03,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:05,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:09,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:09,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:09,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:09,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:10,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:10,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:11,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:14,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:14,576 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:14,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:14,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:15,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:15,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:15,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:16,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:21,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:23,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:23,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:23,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:23,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:23,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:25,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:25,884 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:21:26,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:28,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:28,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:28,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:28,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:29,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:29,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:32,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:33,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:33,187 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:33,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:33,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:33,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:35,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:38,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:38,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:38,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:38,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:38,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:39,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:41,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:41,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:41,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:41,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:41,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:42,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:43,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:43,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:47,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:47,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:47,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:47,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:48,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:49,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:49,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:53,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:53,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:53,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:53,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:54,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:55,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:55,886 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:01,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:02,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:02,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:02,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:02,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:03,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:05,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:06,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:07,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:07,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:07,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:07,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:07,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:09,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:11,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:11,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:11,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:11,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:11,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:12,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:13,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:15,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:17,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:18,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:18,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:18,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:18,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:18,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:19,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:21,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:21,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:21,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:21,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:22,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:22,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:23,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:25,889 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:26,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:26,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:26,035 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:26,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:26,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:27,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:28,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:29,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:29,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:29,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:29,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:30,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:31,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:33,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:33,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:33,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:33,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:33,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:34,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:35,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:37,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:37,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:37,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:37,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:38,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:38,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:39,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:40,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:40,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:40,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:40,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:41,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:43,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:43,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:46,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:46,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:46,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:46,600 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:47,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:47,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:49,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:52,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:52,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:52,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:52,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:52,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:53,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:55,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:55,891 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:56,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:56,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:56,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:57,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:00,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:01,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:01,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:01,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:01,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:03,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:05,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:05,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:05,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:05,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:05,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:06,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:07,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:09,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:09,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:09,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:09,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:09,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:11,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:13,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:13,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:13,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:13,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:14,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:15,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:16,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:18,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:18,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:18,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:18,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:18,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:19,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:22,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:22,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:22,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:22,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:22,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:23,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:23,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:25,900 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:23:26,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:26,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:26,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:26,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:26,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:27,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:30,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:30,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:32,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:32,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:32,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:32,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:32,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:33,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:33,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:37,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:39,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:39,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:39,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:39,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:40,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:41,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:42,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:42,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:42,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:42,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:43,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:43,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:43,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:45,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:45,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:48,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:48,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:48,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:48,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:49,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:49,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:51,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:51,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:51,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:51,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:52,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:53,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:54,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:55,895 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:23:56,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:56,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:56,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:56,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:57,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:57,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:00,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:00,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:00,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:00,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:00,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:01,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:05,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:06,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:06,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:06,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:06,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:07,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:11,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:11,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:11,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:11,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:11,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:11,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:13,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:15,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:15,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:15,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:15,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:15,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:16,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:17,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:19,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:19,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:19,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:19,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:19,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:21,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:22,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:25,902 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:24:26,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:26,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:26,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:26,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:27,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:27,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:27,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:30,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:30,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:31,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:31,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:31,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:32,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:32,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:33,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:33,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:35,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:35,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:35,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:35,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:35,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:37,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:38,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:39,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:39,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:39,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:39,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:40,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:41,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:43,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:45,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:45,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:47,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:47,840 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:47,840 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:47,846 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:48,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:48,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:49,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:51,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:51,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:51,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:53,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:54,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:55,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:55,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:55,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:55,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:55,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:55,899 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:24:57,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:59,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:03,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:03,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:03,682 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:03,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:04,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:05,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:05,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:08,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:08,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:08,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:08,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:08,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:09,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:11,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:15,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:15,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:15,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:15,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:16,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:17,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:17,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:19,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:19,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:19,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:19,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:20,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:21,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:22,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:23,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:23,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:23,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:23,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:23,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:25,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:25,904 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:25:27,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:27,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:27,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:27,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:27,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:28,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:29,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:32,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:32,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:32,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:32,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:32,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:33,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:33,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:36,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:36,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:36,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:36,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:36,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:37,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:39,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:39,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:39,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:39,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:39,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:40,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:41,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:43,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:43,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:43,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:43,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:43,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:45,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:45,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:45,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:48,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:48,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:48,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:48,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:48,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:49,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:50,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:53,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:53,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:53,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:53,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:53,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:55,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:55,903 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:25:55,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:57,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:57,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:57,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:57,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:57,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:59,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:00,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:01,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:01,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:01,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:01,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:01,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:01,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:03,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:06,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:09,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:09,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:09,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:09,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:10,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:11,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:11,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:13,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:13,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:13,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:13,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:13,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:15,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:15,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:15,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:17,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:19,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:19,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:19,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:19,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:20,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:21,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:23,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:23,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:23,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:23,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:23,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:23,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:25,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:25,906 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:26:28,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:28,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:28,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:28,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:28,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:28,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:29,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:32,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:32,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:32,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:32,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:33,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:33,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:33,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:36,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:36,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:36,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:36,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:36,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:37,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:39,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:39,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:39,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:39,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:39,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:39,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:41,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:43,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:43,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:43,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:43,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:43,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:44,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:45,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:45,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:49,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:52,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:52,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:52,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:52,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:53,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:53,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:54,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:55,907 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:26:57,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:57,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:57,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:57,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:57,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:59,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:00,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:01,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:01,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:01,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:01,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:01,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:03,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:04,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:04,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:04,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:04,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:05,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:05,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:05,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:10,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:10,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:10,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:10,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:11,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:11,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:11,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:15,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:15,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:15,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:15,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:15,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:15,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:15,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:17,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:20,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:20,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:20,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:20,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:20,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:21,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:23,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:25,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:27:26,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:26,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:26,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:26,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:26,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:27,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:28,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:32,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:32,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:32,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:32,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:32,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:33,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:34,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:36,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:36,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:36,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:36,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:36,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:37,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:39,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:39,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:39,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:39,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:39,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:41,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:42,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:42,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:42,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:42,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:42,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:43,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:44,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:48,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:48,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:48,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:48,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:49,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:54,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:54,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:54,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:54,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:55,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:55,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:55,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:27:58,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:58,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:58,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:58,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:59,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:59,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:00,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:00,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:05,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:05,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:05,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:05,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:07,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:09,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:09,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:09,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:09,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:09,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:11,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:11,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:14,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:14,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:14,035 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:14,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:14,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:15,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:16,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:22,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:22,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:22,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:22,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:22,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:23,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:25,914 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:28:26,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:26,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:26,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:26,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:26,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:26,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:27,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:30,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:30,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:32,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:34,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:34,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:34,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:34,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:34,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:35,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:38,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:43,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:44,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:44,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:44,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:44,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:44,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:45,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:45,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:48,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:49,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:49,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:49,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:49,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:49,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:51,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:53,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:55,919 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:28:55,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:55,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:55,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:55,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:56,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:57,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:58,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:00,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:00,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:00,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:00,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:00,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:01,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:04,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:07,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:07,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:07,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:07,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:07,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:09,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:10,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:10,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:10,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:10,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:10,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:11,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:14,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:14,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:14,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:14,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:14,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:15,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:15,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:20,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:21,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:21,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:21,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:21,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:21,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:23,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:24,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:24,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:24,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:24,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:25,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:25,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:25,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:25,918 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:29:28,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:28,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:28,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:29,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:30,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:30,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:31,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:33,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:33,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:33,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:33,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:35,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:36,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:37,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:37,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:37,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:37,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:37,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:39,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:42,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:42,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:42,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:42,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:42,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:42,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:43,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:45,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:46,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:46,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:46,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:46,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:46,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:47,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:48,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:50,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:50,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:50,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:50,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:50,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:51,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:54,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:54,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:54,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:54,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:54,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:54,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:55,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:55,920 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:29:58,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:58,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:58,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:58,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:59,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:59,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:59,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:00,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:02,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:02,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:02,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:02,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:03,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:03,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:05,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:09,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:09,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:09,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:09,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:09,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:10,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:11,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:13,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:13,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:13,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:13,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:13,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:15,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:16,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:19,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:19,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:19,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:19,089 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:19,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:19,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:22,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:22,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:22,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:22,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:22,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:22,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:23,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:25,922 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:30:27,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:27,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:27,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:27,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:27,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:27,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:29,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:32,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:33,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:33,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:33,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:33,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:35,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:36,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:36,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:36,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:36,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:36,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:37,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:37,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:42,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:42,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:42,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:42,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:42,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:43,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:43,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:45,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:46,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:46,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:46,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:46,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:46,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:47,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:48,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:50,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:50,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:50,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:50,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:50,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:51,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:53,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:53,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:53,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:53,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:53,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:53,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:55,924 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:30:55,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:56,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:56,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:56,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:56,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:57,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:57,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:58,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:00,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:00,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:00,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:00,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:00,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:01,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:04,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:04,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:04,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:04,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:04,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:04,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:05,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:08,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:08,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:08,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:08,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:08,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:09,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:10,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:12,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:12,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:12,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:12,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:12,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:13,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:16,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:16,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:16,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:16,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:16,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:16,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:17,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:21,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:24,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:24,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:24,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:25,930 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:31:25,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:26,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:28,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:28,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:28,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:28,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:28,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:29,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:32,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:32,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:32,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:32,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:32,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:32,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:34,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:37,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:38,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:38,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:38,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:38,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:39,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:40,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:42,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:42,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:43,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:43,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:44,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:45,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:46,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:46,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:46,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:46,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:48,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:49,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:50,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:50,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:50,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:50,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:51,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:52,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:54,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:54,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:54,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:54,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:54,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:55,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:55,929 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:31:56,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:58,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:58,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:58,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:58,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:59,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:00,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:02,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:02,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:02,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:02,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:03,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:04,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:05,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:10,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:10,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:10,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:10,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:10,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:11,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:14,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:16,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:16,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:16,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:17,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:18,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:19,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:19,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:19,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:19,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:20,045 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:21,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:22,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:25,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:32:26,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:26,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:26,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:26,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:27,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:27,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:28,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:29,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:29,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:29,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:29,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:30,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:32,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:32,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:37,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:38,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:38,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:38,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:38,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:38,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:40,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:42,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:42,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:42,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:42,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:42,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:43,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:44,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:45,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:47,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:47,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:47,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:47,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:47,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:48,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:50,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:51,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:51,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:51,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:51,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:52,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:53,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:54,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:55,932 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:32:56,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:56,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:56,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:56,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:57,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:58,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:59,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:00,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:00,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:00,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:00,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:01,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:02,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:04,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:05,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:05,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:05,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:05,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:06,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:08,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:09,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:09,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:09,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:09,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:10,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:10,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:12,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:13,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:13,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:13,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:13,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:14,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:16,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:16,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:18,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:18,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:18,136 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:18,141 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:19,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:20,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:22,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:23,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:23,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:23,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:23,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:24,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:25,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:33:26,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:27,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:27,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:27,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:27,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:27,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:28,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:30,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:30,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:30,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:30,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:30,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:30,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:31,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:32,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:32,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:33,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:33,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:33,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:33,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:34,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:36,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:37,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:42,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:42,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:42,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:42,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:43,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:43,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:44,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:45,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:45,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:47,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:47,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:47,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:47,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:48,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:49,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:50,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:51,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:51,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:51,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:51,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:52,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:54,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:54,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:55,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:33:56,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:56,288 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:56,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:56,293 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:57,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:58,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:00,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:00,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:00,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:00,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:00,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:00,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:01,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:02,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:03,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:03,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:03,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:03,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:04,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:05,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:06,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:07,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:07,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:07,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:07,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:08,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:10,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:11,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:12,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:12,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:12,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:12,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:12,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:14,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:16,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:20,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:20,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:20,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:20,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:21,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:21,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:22,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:24,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:24,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:24,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:24,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:24,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:25,939 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:34:26,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:26,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:27,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:27,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:27,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:27,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:28,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:30,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:30,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:30,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:31,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:31,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:31,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:31,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:32,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:32,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:34,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:37,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:37,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:37,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:37,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:38,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:38,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:40,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:41,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:41,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:41,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:41,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:41,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:44,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:44,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:45,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:45,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:45,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:45,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:45,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:48,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:48,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:48,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:48,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:48,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:49,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:49,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:50,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:53,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:53,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:53,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:53,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:54,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:54,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:55,941 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:34:56,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:57,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:57,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:57,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:57,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:58,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:00,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:00,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:00,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:03,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:03,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:03,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:03,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:04,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:06,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:06,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:07,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:07,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:07,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:07,798 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:08,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:10,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:11,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:11,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:11,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:11,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:12,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:14,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:15,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:15,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:15,091 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:15,096 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:15,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:16,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:18,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:21,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:21,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:21,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:21,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:21,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:22,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:24,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:25,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:25,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:25,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:25,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:25,943 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:35:26,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:26,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:28,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:29,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:29,711 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:29,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:29,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:30,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:30,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:32,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:32,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:33,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:33,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:33,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:33,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:34,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:36,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:36,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:36,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:36,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:36,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:37,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:37,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:38,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:40,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:40,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:40,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:40,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:42,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:42,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:45,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:46,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:46,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:46,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:46,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:46,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:48,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:48,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:49,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:49,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:49,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:49,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:50,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:52,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:53,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:54,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:54,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:54,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:54,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:55,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:55,949 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:35:56,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:58,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:58,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:58,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:58,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:58,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:59,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:00,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:04,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:05,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:05,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:05,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:05,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:06,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:08,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:09,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:10,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:10,054 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:10,054 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:10,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:10,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:12,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:15,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:15,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:15,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:15,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:15,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:16,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:18,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:18,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:18,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:18,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:18,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:19,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:20,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:20,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:21,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:21,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:21,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:21,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:24,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:25,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:25,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:25,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:25,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:25,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:25,948 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:36:25,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:26,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:31,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:33,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:33,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:33,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:33,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:34,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:36,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:36,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:38,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:38,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:38,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:38,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:39,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:40,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:42,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:46,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:46,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:46,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:46,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:47,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:47,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:48,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:50,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:50,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:50,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:50,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:50,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:52,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:53,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:55,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:55,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:55,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:55,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:55,953 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:36:56,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:58,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:58,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:59,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:59,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:59,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:59,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:00,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:02,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:03,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:03,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:03,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:03,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:03,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:04,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:06,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:08,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:08,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:08,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:08,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:08,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:09,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:10,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:11,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:11,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:11,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:11,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:12,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:14,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:14,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:15,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:16,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:16,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:16,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:16,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:16,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:18,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:20,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:21,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:21,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:21,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:21,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:22,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:24,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:25,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:25,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:25,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:25,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:25,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:25,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:25,952 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:37:28,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:30,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:31,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:31,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:31,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:31,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:31,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:34,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:36,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:39,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:39,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:39,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:39,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:40,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:41,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:42,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:47,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:52,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:53,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:53,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:53,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:53,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:54,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:55,954 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:37:56,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:57,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:57,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:57,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:57,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:57,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:58,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:00,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:01,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:01,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:01,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:01,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:01,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:02,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:03,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:05,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:05,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:05,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:05,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:05,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:06,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:09,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:12,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:12,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:12,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:12,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:13,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:14,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:14,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:17,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:17,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:17,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:20,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:20,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:20,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:20,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:20,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:20,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:21,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:22,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:24,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:24,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:24,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:24,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:25,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:25,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:25,956 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:38:26,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:27,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:27,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:27,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:27,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:28,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:30,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:31,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:31,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:31,521 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:31,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:31,526 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:32,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:34,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:35,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:35,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:35,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:35,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:37,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:38,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:40,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:40,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:40,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:40,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:41,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:42,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:42,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:44,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:44,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:44,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:44,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:44,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:45,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:46,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:48,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:48,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:48,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:48,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:48,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:49,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:50,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:53,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:54,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:54,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:55,958 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:38:56,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:58,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:58,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:58,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:58,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:58,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:59,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:00,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:01,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:01,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:01,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:01,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:02,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:04,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:04,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:05,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:05,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:05,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:05,879 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:06,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:08,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:09,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:14,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:14,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:14,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:14,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:14,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:15,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:16,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:18,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:18,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:18,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:18,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:18,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:20,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:20,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:24,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:24,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:24,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:24,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:24,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:25,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:25,960 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:39:26,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:29,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:29,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:29,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:29,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:29,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:30,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:30,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:32,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:32,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:32,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:32,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:33,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:34,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:35,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:36,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:36,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:38,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:40,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:40,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:40,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:40,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:40,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:41,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:42,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:45,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:45,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:45,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:45,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:45,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:45,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:46,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:48,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:51,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:51,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:51,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:51,750 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:51,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:52,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:54,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:55,962 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:39:56,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:56,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:56,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:56,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:56,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:57,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:58,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:59,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:59,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:59,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:59,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:00,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:02,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:02,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:02,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:02,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:02,845 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:02,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:03,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:04,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:06,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:06,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:06,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:06,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:06,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:07,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:08,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:11,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:11,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:11,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:12,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:13,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:14,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:14,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:14,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:14,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:14,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:15,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:16,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:18,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:18,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:18,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:18,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:18,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:18,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:20,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:23,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:24,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:24,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:24,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:24,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:24,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:25,967 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:40:26,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:28,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:29,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:29,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:29,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:29,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:29,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:30,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:30,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:32,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:32,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:32,317 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:32,322 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:32,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:34,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:34,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:37,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:37,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:37,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:37,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:37,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:38,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:40,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:40,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:40,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:40,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:40,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:41,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:42,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:44,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:44,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:44,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:44,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:45,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:45,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:45,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:46,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:48,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:48,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:48,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:48,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:48,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:50,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:51,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:51,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:51,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:51,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:51,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:52,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:55,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:55,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:55,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:55,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:55,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:55,966 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:40:56,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:58,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:00,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:00,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:00,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:00,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:02,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:02,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:04,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:04,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:04,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:04,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:04,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:06,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:06,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:06,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:06,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:06,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:07,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:07,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:08,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:12,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:12,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:12,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:12,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:12,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:13,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:14,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:15,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:15,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:17,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:17,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:17,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:17,781 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:18,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:18,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:20,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:23,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:24,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:24,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:25,974 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:41:26,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:29,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:30,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:30,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:30,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:32,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:32,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:32,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:32,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:34,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:35,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:35,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:35,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:35,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:35,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:35,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:36,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:39,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:39,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:39,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:39,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:39,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:40,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:41,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:41,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:41,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:41,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:41,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:42,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:44,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:45,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:45,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:46,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:46,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:46,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:46,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:46,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:46,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:48,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:49,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:49,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:49,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:49,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:49,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:51,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:52,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:53,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:53,108 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:53,108 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:53,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:53,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:54,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:55,970 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:41:56,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:57,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:57,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:57,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:57,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:57,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:58,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:00,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:00,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:00,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:00,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:00,714 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:00,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:01,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:02,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:02,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:05,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:05,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:05,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:05,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:08,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:08,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:10,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:10,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:10,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:10,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:10,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:12,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:13,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:13,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:13,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:13,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:13,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:14,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:15,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:15,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:16,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:17,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:17,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:17,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:17,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:17,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:18,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:19,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:21,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:21,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:21,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:21,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:21,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:22,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:25,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:25,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:25,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:25,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:25,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:25,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:25,976 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:42:26,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:28,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:28,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:28,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:28,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:29,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:30,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:32,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:32,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:32,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:32,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:33,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:34,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:35,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:37,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:37,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:37,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:37,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:37,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:38,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:41,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:42,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:42,725 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:42,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:42,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:43,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:44,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:45,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:46,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:47,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:47,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:47,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:47,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:48,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:50,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:51,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:51,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:51,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:51,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:51,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:52,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:54,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:55,975 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:42:56,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:58,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:58,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:58,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:58,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:58,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:00,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:00,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:00,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:00,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:02,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:03,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:03,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:03,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:03,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:03,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:06,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:07,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:10,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:10,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:10,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:10,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:10,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:12,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:13,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:15,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:15,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:18,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:18,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:18,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:18,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:18,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:19,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:20,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:22,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:22,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:22,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:22,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:22,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:23,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:24,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:25,977 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:43:28,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:28,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:28,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:28,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:28,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:29,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:30,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:34,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:34,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:34,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:34,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:34,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:35,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:36,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:37,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:37,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:37,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:37,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:38,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:40,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:40,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:42,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:42,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:42,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:42,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:42,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:44,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:45,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:45,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:45,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:45,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:45,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:46,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:46,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:49,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:49,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:49,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:49,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:49,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:50,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:52,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:52,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:52,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:52,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:52,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:53,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:54,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:55,979 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:43:56,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:56,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:56,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:56,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:56,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:58,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:58,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:59,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:59,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:59,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:59,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:59,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:00,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:00,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:03,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:04,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:04,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:04,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:04,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:04,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:06,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:09,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:09,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:09,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:09,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:09,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:09,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:10,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:12,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:12,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:12,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:12,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:12,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:14,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:14,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:16,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:16,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:16,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:18,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:19,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:22,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:22,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:22,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:22,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:24,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:25,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:25,981 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:44:26,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:26,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:26,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:26,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:26,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:28,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:30,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:30,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:30,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:30,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:32,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:35,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:35,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:35,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:35,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:35,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:36,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:36,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:38,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:38,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:38,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:38,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:39,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:40,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:41,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:42,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:42,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:42,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:42,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:42,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:44,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:47,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:50,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:50,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:50,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:50,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:50,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:52,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:53,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:54,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:54,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:54,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:54,853 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:55,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:55,983 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:44:56,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:58,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:58,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:58,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:58,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:58,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:58,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:00,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:00,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:02,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:02,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:02,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:02,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:02,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:03,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:04,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:06,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:06,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:06,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:06,461 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:06,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:08,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:09,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:12,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:12,273 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:12,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:12,278 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:12,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:14,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:15,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:15,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:15,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:15,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:15,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:16,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:18,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:18,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:18,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:18,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:18,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:20,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:20,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:25,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:25,985 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:45:27,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:27,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:27,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:27,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:27,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:28,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:30,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:30,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:31,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:31,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:31,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:31,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:31,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:32,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:36,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:36,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:36,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:36,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:36,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:37,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:38,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:40,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:40,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:40,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:40,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:40,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:42,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:42,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:44,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:44,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:44,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:44,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:44,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:46,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:47,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:50,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:50,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:50,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:50,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:50,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:52,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:53,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:55,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:55,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:55,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:55,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:55,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:55,990 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:45:56,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:58,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:59,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:59,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:59,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:59,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:59,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:00,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:04,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:07,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:07,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:07,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:07,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:08,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:09,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:10,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:11,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:11,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:11,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:11,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:11,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:12,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:14,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:14,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:14,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:14,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:15,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:16,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:18,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:18,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:18,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:18,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:18,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:20,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:20,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:21,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:21,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:21,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:21,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:22,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:24,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:25,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:25,989 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:46:26,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:26,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:26,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:26,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:27,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:28,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:30,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:31,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:35,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:35,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:35,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:35,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:35,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:36,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:36,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:38,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:38,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:38,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:38,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:38,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:40,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:41,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:42,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:42,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:42,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:42,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:42,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:44,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:45,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:47,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:51,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:51,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:51,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:51,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:51,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:52,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:53,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:55,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:55,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:55,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:55,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:55,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:55,994 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:46:56,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:58,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:58,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:58,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:58,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:58,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:58,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:00,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:03,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:03,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:03,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:03,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:03,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:04,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:04,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:08,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:08,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:08,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:08,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:08,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:10,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:10,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:14,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:14,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:14,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:14,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:14,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:15,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:15,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:16,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:18,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:18,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:18,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:18,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:18,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:20,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:21,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:22,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:22,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:22,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:22,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:22,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:24,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:25,993 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:47:26,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:26,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:26,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:26,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:26,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:27,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:28,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:30,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:32,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:35,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:35,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:35,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:35,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:35,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:36,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:37,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:40,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:40,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:42,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:43,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:44,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:44,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:44,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:44,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:44,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:45,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:46,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:48,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:49,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:49,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:49,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:49,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:49,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:50,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:53,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:53,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:53,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:53,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:53,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:53,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:54,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:55,995 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:47:58,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:02,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:02,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:02,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:02,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:02,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:04,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:04,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:09,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:11,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:11,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:11,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:11,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:12,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:15,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:15,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:15,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:15,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:15,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:16,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:17,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:20,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:20,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:20,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:20,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:20,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:23,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:24,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:24,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:24,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:24,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:25,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:25,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:25,998 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:48:27,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:29,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:29,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:29,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:29,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:30,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:30,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:30,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:31,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:32,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:32,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:32,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:32,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:33,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:35,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:36,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:36,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:36,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:36,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:36,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:39,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:39,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:39,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:39,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:39,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:40,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:41,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:41,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:43,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:43,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:43,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:43,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:44,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:45,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:45,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:45,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:47,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:47,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:47,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:47,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:47,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:48,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:49,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:52,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:56,000 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:48:56,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:56,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:56,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:56,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:57,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:57,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:59,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:00,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:00,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:01,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:01,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:01,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:01,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:01,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:03,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:03,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:07,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:07,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:07,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:07,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:08,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:08,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:09,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:11,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:11,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:11,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:11,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:12,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:13,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:13,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:15,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:15,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:15,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:15,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:16,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:17,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:19,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:19,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:19,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:19,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:19,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:20,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:21,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:23,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:23,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:23,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:23,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:24,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:25,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:25,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:26,005 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:49:28,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:28,245 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:28,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:28,250 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:29,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:30,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:31,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:33,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:33,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:33,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:33,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:34,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:35,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:36,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:37,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:37,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:37,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:37,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:38,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:39,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:41,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:42,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:42,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:42,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:42,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:43,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:45,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:45,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:46,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:46,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:46,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:46,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:47,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:47,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:49,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:49,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:49,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:49,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:49,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:50,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:51,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:52,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:53,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:53,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:53,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:53,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:54,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:55,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:56,004 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:49:56,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:56,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:56,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:56,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:56,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:57,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:58,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:59,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:59,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:59,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:59,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:00,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:00,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:01,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:03,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:05,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:05,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:05,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:05,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:06,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:07,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:09,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:09,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:09,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:09,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:09,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:10,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:11,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:12,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:12,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:12,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:12,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:13,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:14,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:15,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:16,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:16,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:16,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:16,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:17,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:19,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:20,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:20,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:20,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:20,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:20,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:21,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:23,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:24,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:24,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:24,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:24,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:25,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:26,011 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:50:26,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:27,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:27,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:27,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:27,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:27,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:28,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:29,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:31,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:31,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:31,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:31,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:31,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:32,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:33,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:35,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:35,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:35,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:35,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:36,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:36,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:37,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:39,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:39,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:39,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:39,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:40,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:41,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:42,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:46,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:46,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:46,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:46,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:47,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:47,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:49,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:49,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:49,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:49,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:49,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:51,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:52,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:53,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:53,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:53,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:53,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:54,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:55,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:56,008 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:50:57,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:57,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:57,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:57,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:58,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:58,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:59,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:00,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:00,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:00,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:00,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:01,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:03,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:03,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:04,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:04,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:04,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:04,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:05,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:07,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:09,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:09,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:09,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:09,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:09,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:10,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:11,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:13,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:13,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:13,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:13,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:14,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:15,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:15,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:16,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:16,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:16,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:16,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:17,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:19,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:21,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:21,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:21,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:21,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:21,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:22,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:23,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:24,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:24,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:24,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:24,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:25,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:26,013 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:51:26,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:27,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:29,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:29,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:29,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:29,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:30,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:31,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:31,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:33,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:33,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:33,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:33,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:34,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:35,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:37,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:39,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:39,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:39,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:39,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:40,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:41,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:42,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:42,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:42,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:42,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:42,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:43,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:45,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:47,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:47,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:47,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:47,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:47,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:48,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:49,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:51,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:51,485 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:51,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:51,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:52,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:53,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:53,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:54,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:54,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:54,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:54,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:55,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:56,012 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:51:57,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:58,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:58,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:58,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:58,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:59,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:59,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:01,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:02,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:02,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:02,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:02,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:03,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:04,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:05,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:06,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:06,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:06,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:06,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:06,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:07,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:10,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:11,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:11,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:11,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:12,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:13,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:14,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:14,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:14,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:14,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:15,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:15,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:17,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:18,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:18,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:18,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:18,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:18,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:19,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:21,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:22,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:22,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:22,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:22,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:23,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:26,017 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:52:26,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:26,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:26,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:26,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:27,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:27,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:29,159 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:29,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:29,164 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:29,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:31,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:32,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:33,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:33,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:33,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:33,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:34,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:35,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:38,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:38,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:38,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:38,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:38,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:39,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:41,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:43,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:45,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:45,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:45,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:45,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:46,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:47,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:48,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:48,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:48,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:48,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:48,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:49,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:51,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:53,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:53,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:53,629 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:53,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:54,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:54,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:55,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:56,016 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:52:57,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:57,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:57,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:57,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:58,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:59,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:00,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:01,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:01,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:01,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:01,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:01,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:03,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:05,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:05,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:05,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:05,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:06,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:06,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:07,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:10,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:10,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:10,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:10,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:10,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:11,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:12,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:15,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:15,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:15,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:15,777 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:16,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:17,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:17,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:20,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:20,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:20,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:20,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:20,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:21,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:23,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:23,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:23,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:23,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:23,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:24,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:25,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:26,018 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:53:27,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:27,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:27,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:27,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:27,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:29,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:29,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:31,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:31,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:31,999 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:32,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:32,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:33,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:35,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:35,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:35,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:35,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:35,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:36,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:37,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:38,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:38,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:38,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:38,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:39,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:40,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:41,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:41,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:41,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:41,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:41,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:42,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:43,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:45,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:45,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:45,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:45,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:45,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:45,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:47,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:50,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:52,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:52,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:52,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:52,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:53,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:55,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:55,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:56,020 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:53:57,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:57,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:57,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:57,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:58,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:59,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:01,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:01,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:01,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:01,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:01,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:01,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:03,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:05,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:05,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:05,485 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:05,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:06,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:06,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:09,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:09,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:09,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:09,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:09,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:11,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:12,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:14,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:14,193 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:14,193 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:14,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:14,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:15,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:15,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:17,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:18,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:18,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:18,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:18,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:18,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:19,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:21,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:21,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:21,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:21,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:22,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:23,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:26,023 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:54:27,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:27,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:27,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:27,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:27,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:28,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:29,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:30,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:32,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:32,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:32,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:32,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:33,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:34,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:35,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:36,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:36,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:36,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:36,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:37,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:39,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:39,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:40,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:40,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:41,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:43,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:43,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:43,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:43,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:44,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:45,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:46,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:46,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:46,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:46,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:47,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:49,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:51,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:51,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:51,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:51,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:51,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:52,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:53,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:54,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:54,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:54,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:54,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:54,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:55,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:56,025 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:54:56,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:59,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:59,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:00,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:01,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:01,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:03,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:03,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:03,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:03,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:03,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:05,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:06,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:06,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:06,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:06,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:06,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:07,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:09,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:10,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:10,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:10,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:10,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:10,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:11,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:12,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:13,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:13,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:13,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:13,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:14,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:15,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:15,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:17,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:17,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:17,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:17,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:17,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:18,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:19,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:21,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:21,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:21,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:21,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:22,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:22,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:23,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:25,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:25,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:25,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:25,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:25,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:26,027 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:55:27,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:28,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:30,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:30,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:30,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:30,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:30,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:30,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:31,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:33,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:34,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:34,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:34,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:34,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:35,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:37,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:38,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:38,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:38,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:38,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:38,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:39,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:39,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:42,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:42,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:42,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:42,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:42,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:43,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:44,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:49,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:50,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:50,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:50,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:50,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:51,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:53,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:54,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:55,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:55,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:55,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:55,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:56,029 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:55:56,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:57,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:59,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:59,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:59,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:59,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:59,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:00,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:01,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:03,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:03,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:03,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:03,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:04,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:05,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:05,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:08,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:08,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:08,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:08,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:08,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:09,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:11,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:12,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:12,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:12,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:12,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:13,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:15,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:15,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:16,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:21,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:22,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:22,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:22,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:22,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:22,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:23,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:26,031 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:56:27,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:27,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:27,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:27,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:27,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:27,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:30,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:30,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:30,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:30,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:31,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:32,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:33,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:34,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:34,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:34,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:34,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:35,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:37,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:38,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:40,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:40,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:40,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:40,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:41,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:43,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:43,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:46,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:46,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:46,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:46,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:47,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:49,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:49,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:54,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:55,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:55,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:55,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:55,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:55,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:56,037 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:56:57,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:00,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:03,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:03,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:03,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:03,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:03,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:05,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:05,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:06,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:06,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:06,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:06,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:07,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:09,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:10,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:10,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:10,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:10,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:10,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:11,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:11,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:14,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:14,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:14,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:14,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:14,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:15,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:15,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:16,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:18,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:18,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:18,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:18,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:18,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:19,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:22,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:24,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:24,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:24,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:24,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:24,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:25,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:26,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:57:28,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:30,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:30,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:30,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:30,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:30,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:31,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:33,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:34,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:34,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:34,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:34,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:35,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:37,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:38,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:41,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:41,529 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:41,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:41,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:41,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:43,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:44,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:45,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:45,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:45,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:45,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:45,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:45,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:47,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:49,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:53,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:53,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:53,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:53,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:54,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:54,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:55,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:56,040 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:57:58,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:58,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:58,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:58,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:59,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:00,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:01,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:04,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:04,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:04,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:04,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:04,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:05,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:06,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:08,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:08,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:08,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:08,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:09,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:11,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:11,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:11,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:11,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:11,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:12,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:13,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:17,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:20,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:20,184 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:20,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:20,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:20,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:21,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:23,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:23,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:23,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:23,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:23,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:23,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:25,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:26,039 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:58:27,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:27,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:27,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:27,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:28,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:28,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:29,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:31,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:31,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:31,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:31,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:31,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:33,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:34,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:39,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:39,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:39,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:39,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:39,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:39,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:41,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:44,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:45,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:45,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:45,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:45,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:45,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:45,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:47,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:48,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:48,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:48,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:48,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:48,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:49,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:50,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:55,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:56,041 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:58:56,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:56,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:56,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:56,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:57,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:00,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:05,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:05,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:05,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:05,048 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:05,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:06,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:07,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:09,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:09,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:09,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:09,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:09,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:11,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:11,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:15,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:15,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:15,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:15,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:15,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:16,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:17,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:19,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:19,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:19,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:19,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:19,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:21,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:22,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:23,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:23,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:23,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:23,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:23,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:25,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:26,048 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:59:27,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:27,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:27,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:27,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:27,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:27,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:29,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:32,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:34,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:34,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:34,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:34,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:34,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:35,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:37,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:37,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:37,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:37,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:37,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:37,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:39,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:42,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:42,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:42,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:42,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:42,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:43,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:43,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:48,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:50,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:50,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:50,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:50,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:50,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:51,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:53,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:54,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:54,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:54,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:54,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:54,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:55,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:56,046 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:59:58,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:58,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:58,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:58,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:58,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:59,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:01,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:02,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:02,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:02,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:02,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:02,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:03,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:04,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:05,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:05,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:05,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:05,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:06,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:07,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:09,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:10,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:10,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:10,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:11,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:13,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:13,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:13,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:13,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:13,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:15,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:15,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:18,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:18,655 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:18,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:18,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:18,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:19,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:20,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:21,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:21,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:21,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:21,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:22,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:23,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:25,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:25,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:25,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:25,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:25,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:26,051 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:00:26,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:27,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:28,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:28,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:28,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:28,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:28,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:29,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:31,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:32,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:32,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:32,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:32,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:32,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:33,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:35,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:35,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:35,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:35,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:35,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:37,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:37,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:39,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:39,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:39,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:39,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:39,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:41,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:43,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:43,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:43,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:43,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:43,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:44,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:45,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:48,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:48,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:48,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:48,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:48,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:48,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:49,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:52,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:52,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:52,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:52,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:52,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:53,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:53,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:56,050 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:00:56,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:56,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:56,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:56,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:57,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:57,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:58,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:02,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:02,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:02,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:02,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:02,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:03,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:04,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:05,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:05,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:05,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:05,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:05,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:07,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:09,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:10,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:10,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:10,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:10,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:10,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:11,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:14,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:14,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:14,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:14,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:14,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:15,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:15,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:18,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:18,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:18,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:18,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:18,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:19,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:21,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:21,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:21,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:21,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:21,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:21,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:23,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:25,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:25,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:25,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:25,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:25,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:26,055 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:01:27,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:27,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:28,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:28,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:28,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:28,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:29,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:31,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:32,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:32,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:32,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:32,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:32,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:32,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:33,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:37,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:38,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:38,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:38,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:38,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:38,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:39,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:41,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:41,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:41,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:41,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:41,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:42,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:43,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:48,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:50,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:50,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:50,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:50,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:51,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:53,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:53,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:53,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:53,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:53,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:53,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:55,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:56,054 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:01:57,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:57,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:57,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:57,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:59,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:59,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:04,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:05,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:05,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:05,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:05,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:06,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:07,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:08,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,970 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:08,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:09,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:11,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:14,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:14,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:14,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:14,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:14,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:15,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:15,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:15,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:19,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:19,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:19,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:19,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:19,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:20,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:21,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:22,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:22,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:22,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:22,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:23,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:23,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:25,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:26,059 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:02:26,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:26,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:26,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:26,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:26,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:27,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:29,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:29,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:29,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:29,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:30,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:30,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:31,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:31,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:35,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:35,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:35,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:35,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:35,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:37,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:37,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:39,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:39,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:39,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:39,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:39,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:41,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:42,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:45,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:45,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:45,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:47,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:48,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:48,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:48,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:48,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:48,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:48,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:49,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:53,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:56,058 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:02:56,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:56,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:56,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:56,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:58,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:00,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:00,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:00,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:00,542 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:00,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:00,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:01,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:04,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:04,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:04,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:04,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:04,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:04,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:05,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:07,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:07,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:07,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:07,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:07,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:09,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:09,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:13,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:13,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:13,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:13,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:13,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:15,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:15,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:15,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:16,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:16,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:16,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:16,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:17,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:19,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:19,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:19,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:19,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:19,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:20,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:21,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:24,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:24,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:24,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:24,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:25,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:25,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:26,061 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:03:28,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:28,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:28,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:28,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:28,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:29,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:30,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:30,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:32,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:32,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:32,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:32,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:33,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:33,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:36,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:36,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:36,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:36,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:36,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:37,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:39,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:39,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:39,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:39,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:39,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:41,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:41,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:43,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:43,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:43,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:43,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:43,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:45,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:47,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:47,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:47,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:47,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:47,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:47,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:49,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:51,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:51,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:51,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:51,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:51,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:52,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:53,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:54,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:54,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:54,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:54,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:54,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:55,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:56,063 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:03:58,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:01,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:01,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:01,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:01,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:01,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:03,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:03,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:04,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:04,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:04,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:04,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:04,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:05,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:08,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:09,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:09,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:09,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:09,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:09,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:11,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:13,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:13,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:13,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:13,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:13,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:13,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:15,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:16,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:16,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:16,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:16,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:16,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:17,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:18,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:20,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:20,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:20,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:20,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:21,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:23,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:23,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:23,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:23,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:23,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:23,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:25,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:26,064 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:04:29,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:29,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:29,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:29,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:29,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:30,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:32,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:33,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:33,584 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:34,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:34,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:36,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:36,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:36,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:36,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:36,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:37,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:38,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:39,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:40,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:40,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:40,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:40,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:41,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:42,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:44,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:47,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:47,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:47,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:47,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:48,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:50,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:50,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:51,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:51,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:51,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:51,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:52,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:52,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:55,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:55,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:55,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:55,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:55,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:56,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:56,066 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:04:58,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:59,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:59,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:59,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:59,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:00,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:01,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:02,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:03,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:03,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:03,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:03,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:04,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:04,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:06,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:06,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:06,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:06,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:07,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:08,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:09,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:09,803 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:09,803 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:09,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:10,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:12,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:12,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:13,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:13,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:13,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:13,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:14,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:16,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:18,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:22,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:22,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:22,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:22,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:23,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:24,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:24,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:26,068 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:05:27,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:27,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:27,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:27,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:28,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:29,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:30,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:30,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:30,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:30,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:30,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:31,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:32,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:33,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:33,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:33,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:33,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:34,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:34,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:36,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:37,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:37,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:37,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:37,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:38,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:38,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:40,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:41,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:41,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:41,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:41,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:42,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:44,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:45,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:45,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:45,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:45,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:45,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:45,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:46,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:49,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:49,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:49,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:49,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:50,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:50,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:55,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:55,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:55,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:55,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:56,074 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:05:56,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:56,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:58,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:58,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:58,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:58,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:58,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:59,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:00,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:01,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:02,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:02,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:02,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:02,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:03,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:04,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:05,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:05,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:05,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:05,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:06,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:06,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:08,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:11,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:11,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:11,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:12,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:12,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:12,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:15,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:15,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:15,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:15,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:16,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:17,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:18,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:19,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:19,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:19,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:19,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:20,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:22,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:22,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:23,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:23,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:23,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:23,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:24,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:26,073 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:06:26,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:27,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:27,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:27,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:27,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:27,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:28,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:28,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:30,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:30,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:30,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:30,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:31,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:32,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:33,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:35,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:35,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:35,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:35,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:36,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:38,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:39,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:39,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:39,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:39,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:39,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:40,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:42,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:42,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:42,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:42,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:42,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:43,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:44,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:44,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:45,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:48,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:48,209 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:48,209 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:48,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:49,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:50,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:50,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:51,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:51,681 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:51,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:51,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:52,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:54,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:55,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:55,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:55,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:55,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:55,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:56,078 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:06:56,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:58,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:59,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:59,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:59,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:59,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:00,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:01,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:02,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:03,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:03,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:03,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:03,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:04,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:06,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:07,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:09,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:09,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:09,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:09,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:09,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:10,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:12,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:12,236 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:12,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:12,242 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:13,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:13,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:14,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:16,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:16,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:16,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:16,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:18,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:18,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:21,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:21,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:21,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:21,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:21,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:22,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:24,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:26,077 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:07:26,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:26,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:26,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:26,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:27,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:28,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:29,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:30,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:30,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:30,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:30,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:31,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:32,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:34,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:35,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:35,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:35,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:35,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:36,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:38,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:38,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:38,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:38,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:38,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:39,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:40,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:43,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:43,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:43,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:43,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:44,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:46,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:46,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:46,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:46,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:46,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:47,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:48,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:50,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:52,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:52,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:52,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:52,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:53,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:54,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:56,085 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:07:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:59,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:59,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:59,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:59,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:01,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:02,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:02,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:02,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:02,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:03,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:04,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:06,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:06,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:06,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:06,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:07,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:07,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:08,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:10,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:10,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:11,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:12,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:12,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:15,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:17,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:17,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:17,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:17,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:18,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:18,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:18,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:20,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:20,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:20,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:21,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:22,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:23,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:24,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:24,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:24,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:24,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:25,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:26,081 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:08:26,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:29,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:30,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:30,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:30,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:32,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:34,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:34,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:34,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:34,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:34,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:35,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:36,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:38,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:38,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:38,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:38,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:39,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:39,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:40,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:44,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:44,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:44,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:44,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:44,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:45,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:46,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:48,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:48,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:48,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:48,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:49,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:49,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:50,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:52,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:52,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:52,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:52,860 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:53,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:54,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:54,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:56,086 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:08:56,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:56,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:56,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:56,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:58,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:00,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:01,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:01,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:01,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:01,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:02,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:02,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:04,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:04,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:04,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:04,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:05,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:06,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:08,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:08,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:10,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:11,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:11,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:11,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:11,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:11,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:14,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:15,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:15,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:15,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:15,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:15,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:16,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:16,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:18,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:19,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:19,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:19,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:19,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:20,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:22,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:22,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:23,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:23,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:23,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:23,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:24,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:24,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:26,086 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:09:26,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:26,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:26,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:26,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:27,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:28,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:28,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:30,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:30,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:30,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:30,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:31,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:32,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:34,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:35,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:35,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:35,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:36,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:38,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:38,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:38,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:39,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:39,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:40,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:44,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:46,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:46,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:46,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:46,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:47,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:48,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:49,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:49,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:49,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:49,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:50,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:50,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:52,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:53,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:53,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:53,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:53,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:54,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:54,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:56,088 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:09:56,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:56,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:56,739 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:56,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:56,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:57,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:58,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:59,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:59,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:59,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:59,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:00,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:01,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:02,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:04,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:04,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:04,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:04,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:05,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:06,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:07,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:09,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:09,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:09,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:09,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:09,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:10,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:12,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:12,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:12,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:12,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:12,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:13,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:14,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:15,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:15,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:15,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:15,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:15,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:16,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:17,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:18,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:19,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:19,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:19,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:19,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:22,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:23,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:24,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:24,350 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:24,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:24,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:24,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:26,090 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:10:26,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:29,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:30,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:32,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:32,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:32,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:32,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:33,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:34,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:34,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:37,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:37,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:37,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:37,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:37,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:38,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:40,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:41,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:41,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:41,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:41,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:42,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:44,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:45,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:45,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:45,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:45,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:45,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:45,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:46,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:48,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:50,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:50,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:50,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:50,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:50,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:51,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:52,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:56,092 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:10:56,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:56,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:56,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:56,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:56,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:57,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:58,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:00,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:00,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:00,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:00,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:01,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:02,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:04,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:04,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:04,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:04,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:05,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:06,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:07,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:08,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:08,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:08,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:08,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:08,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:10,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:13,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:13,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:13,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:13,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:13,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:14,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:15,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:16,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:18,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:22,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:22,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:22,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:22,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:22,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:24,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:24,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:25,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:25,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:25,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:25,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:26,097 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:11:26,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:29,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:29,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:29,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:29,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:29,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:30,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:30,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:30,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:33,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:33,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:33,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:33,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:34,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:35,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:40,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:41,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:41,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:41,855 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:41,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:42,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:44,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:45,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:45,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:45,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:45,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:45,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:45,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:46,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:46,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:49,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:49,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:49,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:49,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:50,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:50,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:52,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:53,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:53,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:53,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:53,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:54,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:56,096 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:11:56,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:57,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:57,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:57,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:57,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:00,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:01,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:02,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:02,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:02,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:02,914 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:03,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:04,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:06,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:07,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:07,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:07,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:07,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:07,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:08,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:12,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:12,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:12,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:12,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:12,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:12,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:14,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:16,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:16,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:16,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:16,089 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:16,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:18,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:18,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:19,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:19,329 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:19,329 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:19,334 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:19,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:20,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:22,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:22,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:22,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:22,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:23,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:23,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:24,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:26,101 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:12:26,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:26,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:26,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:26,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:26,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:28,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:29,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:32,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:32,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:32,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:32,731 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:33,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:34,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:34,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:36,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:36,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:36,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:36,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:37,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:38,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:40,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:40,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:40,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:40,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:40,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:40,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:42,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:45,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:46,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:46,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:46,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:46,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:47,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:48,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:50,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:50,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:50,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:50,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:50,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:51,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:52,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:55,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:55,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:55,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:56,100 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:12:56,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:58,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:59,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:59,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:00,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:00,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:01,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:02,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:04,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:04,281 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:04,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:04,286 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:04,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:06,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:07,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:12,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:13,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:13,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:13,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:13,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:13,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:14,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:17,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:17,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:17,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:17,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:17,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:17,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:18,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:21,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:21,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:21,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:21,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:22,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:22,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:24,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:24,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:24,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:24,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:24,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:25,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:26,105 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:13:26,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:28,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:33,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:33,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:33,407 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:33,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:33,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:33,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:34,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:38,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:38,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:38,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:38,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:38,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:39,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:40,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:43,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:43,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:43,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:43,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:44,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:44,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:44,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:49,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:49,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:49,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:49,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:50,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:50,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:50,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:52,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:52,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:52,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:52,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:53,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:54,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:56,104 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:13:56,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:56,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:56,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:56,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:57,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:58,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:00,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:01,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:01,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:01,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:01,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:01,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:02,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:05,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:05,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:05,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:05,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:05,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:06,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:06,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:09,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:09,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:09,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:09,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:10,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:10,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:11,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:12,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:12,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:12,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:12,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:13,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:14,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:16,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:16,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:16,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:16,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:16,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:17,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:18,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:20,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:20,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:20,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:20,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:21,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:21,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:22,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:24,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:24,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:24,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:24,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:26,109 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:14:26,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:27,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:27,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:27,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:27,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:27,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:28,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:28,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:31,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:31,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:31,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:31,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:32,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:33,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:34,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:37,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:37,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:37,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:37,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:38,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:38,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:38,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:45,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:45,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:45,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:45,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:45,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:46,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:49,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:53,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:53,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:53,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:54,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:54,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:56,108 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:14:56,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:57,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:57,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:57,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:57,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:58,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:00,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:00,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:00,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:00,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:00,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:01,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:02,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:05,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:05,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:05,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:05,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:05,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:06,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:06,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:11,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:11,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:11,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:11,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:11,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:11,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:12,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:14,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:14,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:14,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:14,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:15,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:16,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:16,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:18,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:18,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:18,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:18,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:19,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:20,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:21,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:23,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:23,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:23,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:23,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:23,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:24,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:26,110 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:15:27,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:27,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:27,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:27,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:27,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:28,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:30,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:30,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:32,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:36,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:36,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:36,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:36,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:36,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:38,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:38,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:39,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:39,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:40,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:40,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:43,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:43,025 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:43,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:43,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:44,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:44,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:45,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:48,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:48,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:48,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:48,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:48,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:49,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:50,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:52,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:52,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:52,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:52,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:52,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:54,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:54,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:56,113 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:00,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:00,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:00,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:00,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:00,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:02,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:04,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:04,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:04,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:04,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:05,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:06,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:08,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:08,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:08,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:08,060 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:08,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:10,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:11,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:15,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:15,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:15,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:15,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:15,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:15,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:16,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:16,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:18,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:18,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:18,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:18,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:19,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:20,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:21,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:22,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:22,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:22,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:22,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:22,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:24,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:26,115 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:26,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:26,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:26,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:26,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:26,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:27,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:28,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:30,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:30,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:30,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:30,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:30,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:32,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:33,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:34,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:34,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:34,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:34,526 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:34,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:36,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:39,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:39,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:39,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:39,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:39,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:39,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:42,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:42,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:42,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:42,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:43,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:44,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:44,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:45,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:46,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:46,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:46,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:46,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:48,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:49,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:50,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:50,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:50,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:50,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:50,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:52,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:54,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:54,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:54,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:54,286 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:54,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:55,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:56,122 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:56,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:57,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:57,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:57,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:57,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:58,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:58,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:00,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:02,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:02,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:02,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:02,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:02,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:04,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:06,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:06,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:06,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:06,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:06,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:06,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:08,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:09,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:09,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:09,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:09,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:10,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:10,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:11,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:14,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:14,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:14,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:14,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:15,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:15,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:16,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:17,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:19,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:19,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:19,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:19,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:19,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:20,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:22,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:23,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:23,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:23,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:23,190 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:23,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:24,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:26,119 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:17:28,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:28,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:28,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:28,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:28,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:28,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:30,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:32,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:32,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:32,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:32,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:33,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:33,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:34,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:37,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:37,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:37,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:37,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:38,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:38,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:39,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:42,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:42,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:42,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:42,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:42,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:44,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:45,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:46,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:46,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:46,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:46,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:46,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:48,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:50,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:50,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:50,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:50,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:51,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:52,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:55,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:56,124 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:17:59,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:59,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:59,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:59,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:59,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:00,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:00,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:01,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:03,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:03,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:03,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:03,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:03,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:04,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:07,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:08,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:08,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:08,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:08,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:09,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:10,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:12,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:12,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:12,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:12,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:12,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:13,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:14,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:15,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:15,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:15,704 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:15,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:15,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:15,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:16,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:18,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:19,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:19,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:19,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:19,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:19,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:20,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:24,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:26,123 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:18:26,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:26,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:26,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:26,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:28,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:29,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:29,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:29,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:29,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:29,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:30,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:30,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:33,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:33,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:33,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:33,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:34,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:34,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:34,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:38,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:38,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:38,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:38,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:38,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:40,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:40,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:43,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:43,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:43,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:43,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:43,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:44,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:45,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:46,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:46,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:46,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:46,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:46,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:48,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:50,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:52,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:53,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:53,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:54,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:56,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:56,128 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:18:56,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:56,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:56,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:56,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:56,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:00,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:00,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:00,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:00,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:01,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:02,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:04,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:04,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:04,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:04,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:04,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:06,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:07,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:12,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:12,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:12,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:12,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:12,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:12,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:14,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:15,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:16,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:16,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:16,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:16,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:16,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:18,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:18,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:19,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:19,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:19,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:19,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:19,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:20,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:23,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:23,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:23,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:23,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:23,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:23,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:24,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:26,127 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:19:26,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:26,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:26,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:26,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:27,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:28,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:28,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:30,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:30,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:30,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:30,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:30,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:32,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:33,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:33,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:33,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:33,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:33,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:34,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:34,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:38,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:42,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:42,562 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:42,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:42,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:42,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:44,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:44,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:45,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:45,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:45,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:45,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:45,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:46,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:49,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:49,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:49,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:49,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:49,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:50,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:53,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:53,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:53,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:53,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:53,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:54,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:55,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:56,129 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:19:56,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:56,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:56,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:56,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:56,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:58,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:00,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:00,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:02,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:04,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:04,029 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:04,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:04,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:04,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:04,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:06,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:08,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:08,390 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:08,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:08,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:08,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:10,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:11,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:12,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:12,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:12,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:12,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:12,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:14,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:16,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:16,335 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:16,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:16,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:16,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:17,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:18,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:22,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:22,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:22,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:22,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:22,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:22,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:24,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:26,131 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:20:26,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:26,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:26,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:26,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:26,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:27,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:28,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:33,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:36,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:36,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:36,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:36,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:36,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:39,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:39,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:39,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:39,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:39,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:39,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:41,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:42,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:42,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:42,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:42,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:43,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:44,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:45,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:45,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:45,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:45,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:45,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:45,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:46,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:47,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:49,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:50,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:50,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:50,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:50,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:51,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:51,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:53,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:53,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:53,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:53,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:54,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:55,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:55,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:56,133 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:20:58,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:58,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:58,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:58,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:59,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:59,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:00,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:01,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:01,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:01,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:01,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:02,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:03,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:05,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:10,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:10,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:10,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:10,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:11,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:11,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:13,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:13,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:13,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:13,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:13,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:14,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:15,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:16,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:19,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:19,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:19,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:19,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:20,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:21,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:21,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:23,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:23,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:23,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:23,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:24,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:25,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:26,135 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:21:27,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:28,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:28,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:28,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:28,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:29,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:31,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:31,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:31,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:31,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:31,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:32,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:32,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:33,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:37,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:38,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:39,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:41,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:41,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:41,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:41,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:41,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:42,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:42,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:43,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:45,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:45,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:45,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:45,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:46,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:47,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:48,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:53,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:54,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:55,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:56,137 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:21:58,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:58,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:58,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:58,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:59,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:59,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:01,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:01,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:01,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:01,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:01,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:02,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:03,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:04,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:06,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:06,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:06,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:06,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:07,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:09,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:10,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:13,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:13,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:13,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:13,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:14,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:15,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:15,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:16,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:19,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:19,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:19,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:19,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:20,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:21,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:22,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:22,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:22,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:22,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:22,453 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:23,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:25,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:26,143 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:22:28,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:30,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:30,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:30,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:30,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:31,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:33,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:33,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:34,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:34,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:34,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:35,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:37,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:39,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:39,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:39,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:39,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:39,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:40,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:41,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:42,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:42,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:42,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:42,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:44,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:45,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:46,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:46,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:46,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:46,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:47,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:49,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:49,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:50,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:50,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:50,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:50,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:51,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:53,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:54,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:56,142 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:22:59,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:59,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:59,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:59,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:00,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:00,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:00,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:01,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:05,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:07,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:07,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:07,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:07,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:08,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:09,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:10,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:15,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:15,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:16,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:17,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:18,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:18,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:18,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:18,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:19,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:21,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:21,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:22,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:22,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:22,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:22,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:22,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:23,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:26,150 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:23:27,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:27,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:27,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:27,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:27,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:28,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:29,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:30,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:30,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:30,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:30,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:31,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:32,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:34,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:34,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:34,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:34,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:35,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:37,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:38,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:38,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:38,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:38,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:38,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:42,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:42,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:42,008 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:42,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:42,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:43,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:43,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:45,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:45,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:45,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:47,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:48,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:50,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:50,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:50,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:50,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:51,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:51,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:54,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:54,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:54,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:54,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:54,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:55,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:55,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:56,146 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:23:59,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:59,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:59,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:59,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:59,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:00,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:00,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:01,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:02,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:02,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:02,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:02,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:03,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:05,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:05,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:07,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:07,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:07,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:07,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:09,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:10,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:11,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:11,472 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:11,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:11,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:12,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:13,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:15,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:15,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:15,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:15,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:15,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:16,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:18,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:18,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:18,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:18,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:19,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:21,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:21,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:22,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:22,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:22,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:22,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:22,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:23,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:26,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:26,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:26,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:26,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:24:26,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:27,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:27,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:30,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:32,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:34,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:34,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:34,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:34,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:35,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:37,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:37,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:37,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:37,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:37,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:37,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:38,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:39,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:43,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:43,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:43,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:43,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:43,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:45,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:48,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:48,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:48,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:48,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:48,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:48,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:49,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:53,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:55,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:55,029 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:55,029 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:55,034 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:55,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:56,150 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:24:57,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:59,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:59,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:59,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:59,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:59,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:00,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:01,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:02,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:02,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:02,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:02,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:03,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:04,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:05,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:06,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:06,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:06,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:06,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:07,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:07,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:10,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:14,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:14,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:14,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:14,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:15,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:15,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:15,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:17,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:18,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:18,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:18,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:18,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:19,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:21,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:24,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:24,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:24,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:24,058 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:24,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:25,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:26,152 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:25:27,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:28,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:31,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:32,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:33,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:33,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:33,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:33,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:34,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:35,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:37,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:37,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:37,553 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:37,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:37,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:38,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:39,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:42,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:42,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:42,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:42,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:42,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:43,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:43,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:45,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:45,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:45,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:45,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:45,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:46,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:47,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:48,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:50,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:50,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:50,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:50,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:51,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:51,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:53,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:53,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:53,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:53,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:53,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:54,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:55,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:56,154 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:25:59,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:02,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:02,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:02,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:02,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:03,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:03,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:04,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:05,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:05,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:05,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:05,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:06,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:07,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:09,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:09,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:09,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:09,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:09,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:10,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:11,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:13,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:13,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:13,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:13,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:14,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:14,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:15,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:19,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:19,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:19,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:19,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:19,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:20,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:21,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:22,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:22,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:22,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:22,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:23,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:25,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:25,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:26,156 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:26:26,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:26,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:26,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:26,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:27,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:27,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:30,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:30,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:30,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:31,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:31,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:31,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:34,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:34,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:34,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:34,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:34,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:35,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:37,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:38,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:38,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:38,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:38,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:39,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:39,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:42,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:42,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:42,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:42,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:43,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:45,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:47,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:52,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:52,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:52,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:52,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:52,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:53,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:53,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:56,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:56,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:56,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:56,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:56,162 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:26:56,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:57,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:59,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:59,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:59,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:59,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:59,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:00,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:01,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:03,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:03,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:03,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:03,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:03,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:04,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:05,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:09,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:09,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:09,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:09,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:09,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:10,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:11,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:12,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:12,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:12,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:12,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:13,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:15,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:15,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:16,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:16,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:16,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:16,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:17,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:17,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:20,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:22,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:22,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:22,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:22,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:22,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:23,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:26,160 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:27:26,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:26,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:26,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:26,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:26,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:27,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:29,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:30,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:30,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:30,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:31,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:31,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:31,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:34,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:34,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:34,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:34,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:34,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:35,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:37,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:39,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:39,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:39,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:39,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:40,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:41,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:42,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:43,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:43,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:43,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:43,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:43,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:45,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:45,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:47,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:47,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:47,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:47,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:47,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:48,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:49,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:52,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:54,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:54,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:54,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:54,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:54,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:55,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:56,166 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:27:57,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:57,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:57,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:57,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:57,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:58,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:59,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:01,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:01,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:01,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:01,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:01,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:03,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:03,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:05,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:05,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:05,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:05,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:06,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:07,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:08,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:10,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:10,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:10,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:10,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:10,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:11,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:13,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:13,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:13,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:13,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:13,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:14,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:15,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:19,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:19,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:19,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:19,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:19,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:20,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:21,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:22,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:22,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:22,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:22,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:23,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:25,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:25,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:26,164 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:28:26,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:26,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:26,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:26,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:30,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:30,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:30,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:30,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:31,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:31,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:35,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:35,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:35,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:35,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:36,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:36,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:37,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:39,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:39,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:39,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:39,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:39,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:41,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:42,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:45,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:45,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:45,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:45,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:45,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:47,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:47,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:48,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:48,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:48,759 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:48,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:49,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:51,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:52,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:52,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:52,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:52,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:52,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:53,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:53,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:55,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:55,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:55,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:55,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:55,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:56,170 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:28:57,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:59,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:59,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:59,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:59,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:59,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:00,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:00,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:01,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:04,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:07,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:07,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:07,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:07,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:07,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:09,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:10,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:10,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:10,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:10,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:10,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:11,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:13,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:14,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:14,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:14,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:14,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:15,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:15,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:15,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:18,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:18,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:18,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:18,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:18,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:19,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:21,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:22,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:22,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:22,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:22,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:22,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:23,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:25,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:25,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:25,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:25,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:26,169 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:29:26,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:27,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:27,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:30,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:32,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:35,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:35,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:35,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:35,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:36,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:37,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:37,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:39,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:39,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:39,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:39,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:39,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:41,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:43,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:44,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:44,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:44,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:44,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:44,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:45,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:48,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:49,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:49,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:49,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:49,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:49,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:51,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:53,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:53,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:53,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:53,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:53,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:54,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:55,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:56,171 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:29:59,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:59,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:59,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:59,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:59,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:59,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:00,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:01,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:03,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:03,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:03,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:03,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:04,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:04,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:05,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:07,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:07,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:07,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:07,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:07,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:09,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:10,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:12,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:12,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:12,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:12,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:12,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:13,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:15,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:15,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:16,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:16,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:16,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:17,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:20,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:20,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:20,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:20,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:20,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:21,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:21,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:24,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:24,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:24,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:24,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:25,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:25,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:26,173 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:30:27,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:29,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:29,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:29,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:29,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:29,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:31,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:32,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:37,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:38,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:38,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:38,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:38,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:39,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:42,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:42,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:42,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:42,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:42,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:43,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:43,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:48,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:51,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:51,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:51,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:51,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:51,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:53,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:54,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:55,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:55,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:55,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:55,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:56,175 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:30:56,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:57,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:59,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:59,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:59,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:59,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:59,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:00,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:01,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:03,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:03,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:03,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:03,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:04,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:05,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:05,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:08,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:08,236 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:08,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:08,241 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:08,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:09,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:11,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:15,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:15,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:15,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:15,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:15,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:16,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:17,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:19,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:19,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:19,562 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:19,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:19,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:21,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:22,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:26,180 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:31:28,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:28,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:28,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:28,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:28,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:28,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:29,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:31,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:31,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:31,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:31,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:31,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:33,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:33,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:35,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:35,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:35,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:35,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:35,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:37,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:39,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:41,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:41,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:41,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:41,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:41,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:43,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:44,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:45,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:45,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:45,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:47,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:49,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:50,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:50,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:50,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:50,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:50,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:51,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:53,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:53,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:53,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:53,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:53,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:55,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:55,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:56,179 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:31:58,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:58,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:58,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:58,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:58,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:59,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:00,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:02,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:02,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:02,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:02,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:02,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:03,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:06,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:06,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:06,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:06,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:06,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:06,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:07,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:10,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:10,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:10,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:10,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:10,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:11,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:12,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:13,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:13,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:13,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:13,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:14,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:15,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:15,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:17,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:17,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:17,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:17,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:17,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:17,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:19,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:21,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:21,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:21,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:21,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:22,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:23,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:24,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:24,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:24,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:24,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:24,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:25,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:26,184 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:32:28,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:28,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:28,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:28,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:28,644 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:28,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:29,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:32,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:32,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:32,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:32,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:32,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:33,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:34,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:36,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:36,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:36,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:36,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:36,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:37,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:39,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:39,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:39,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:39,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:39,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:39,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:41,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:42,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:42,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:42,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:42,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:43,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:44,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:45,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:45,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:45,886 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:45,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:46,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:47,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:49,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:49,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:49,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:49,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:49,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:50,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:51,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:52,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:52,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:52,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:52,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:52,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:53,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:55,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:56,183 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:32:57,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:57,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:57,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:57,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:57,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:59,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:00,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:00,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:00,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:00,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:00,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:01,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:01,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:03,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:03,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:03,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:03,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:04,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:05,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:05,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:07,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:07,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:07,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:07,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:07,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:09,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:11,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:13,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:13,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:13,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:13,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:13,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:15,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:16,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:17,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:17,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:17,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:17,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:17,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:19,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:21,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:21,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:21,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:21,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:21,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:22,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:23,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:25,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:25,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:25,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:25,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:26,188 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:33:27,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:27,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:28,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:28,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:28,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:28,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:28,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:29,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:31,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:31,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:31,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:31,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:32,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:32,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:33,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:34,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:34,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:34,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:34,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:35,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:37,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:38,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:38,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:38,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:38,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:39,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:39,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:43,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:43,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:43,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:43,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:43,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:43,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:45,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:49,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:50,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:50,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:50,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:50,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:50,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:51,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:54,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:54,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:54,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:54,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:54,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:55,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:55,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:56,187 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:33:58,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:58,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:58,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:58,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:58,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:59,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:00,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:00,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:03,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:03,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:03,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:03,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:03,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:05,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:06,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:07,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:07,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:07,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:07,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:07,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:09,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:11,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:11,373 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:11,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:11,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:11,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:12,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:13,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:14,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:14,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:14,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:14,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:15,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:17,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:17,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:20,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:20,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:20,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:20,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:20,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:21,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:23,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:25,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:25,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:25,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:25,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:25,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:26,195 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:34:27,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:28,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:28,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:28,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:28,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:28,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:28,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:29,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:33,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:33,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:33,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:33,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:33,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:34,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:35,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:39,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:41,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:41,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:41,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:41,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:41,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:43,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:44,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:45,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:45,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:45,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:45,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:45,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:47,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:49,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:49,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:49,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:49,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:49,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:50,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:51,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:52,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:52,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:52,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:52,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:52,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:55,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:55,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:56,192 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:34:56,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:56,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:56,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:56,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:56,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:59,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:00,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:00,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:00,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:00,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:00,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:01,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:01,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:03,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:04,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:04,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:04,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:04,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:04,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:05,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:09,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:09,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:09,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:09,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:11,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:11,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:14,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:14,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:14,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:14,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:14,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:15,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:16,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:18,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:18,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:18,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:18,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:18,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:19,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:21,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:21,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:21,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:21,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:21,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:21,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:23,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:24,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:24,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:24,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:24,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:24,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:25,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:26,197 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:35:27,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:27,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:27,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:27,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:27,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:27,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:29,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:32,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:32,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:32,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:32,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:32,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:33,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:33,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:35,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:35,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:35,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:35,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:35,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:37,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:38,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:39,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:39,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:39,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:39,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:41,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:44,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:44,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:44,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:44,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:44,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:44,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:45,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:49,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:50,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:50,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:50,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:50,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:50,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:51,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:54,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:54,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:54,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:54,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:54,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:55,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:56,196 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:35:59,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:59,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:59,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:59,410 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:59,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:00,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:01,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:04,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:04,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:04,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:05,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:06,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:07,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:07,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:07,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:07,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:07,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:09,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:11,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:12,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:12,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:12,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:12,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:12,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:13,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:15,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:15,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:15,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:15,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:15,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:17,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:17,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:22,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:24,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:24,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:24,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:24,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:24,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:25,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:26,198 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:36:28,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:29,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:29,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:29,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:31,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:32,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:32,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:32,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:32,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:32,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:33,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:33,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:36,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:36,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:36,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:36,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:36,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:37,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:39,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:42,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:42,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:42,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:42,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:42,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:43,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:45,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:45,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:45,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:47,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:51,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:51,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:51,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:51,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:51,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:51,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:53,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:56,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:56,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:56,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:56,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:56,200 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:36:56,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:58,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:00,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:00,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:00,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:00,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:01,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:01,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:02,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:04,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:04,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:04,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:04,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:05,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:06,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:07,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:12,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:12,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:12,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:12,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:12,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:13,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:16,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:16,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:16,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:16,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:16,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:17,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:18,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:18,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:20,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:20,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:20,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:20,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:21,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:22,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:23,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:26,202 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:37:27,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:27,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:27,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:27,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:27,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:29,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:30,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:33,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:33,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:33,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:33,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:34,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:34,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:36,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:37,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:37,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:37,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:38,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:38,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:40,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:40,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:43,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:43,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:43,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:44,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:46,045 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:46,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:46,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:46,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:46,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:47,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:48,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:49,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:49,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:49,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:49,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:50,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:50,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:52,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:53,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:53,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:53,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:53,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:54,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:55,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:56,207 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:37:57,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:57,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:57,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:57,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:58,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:00,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:01,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:02,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:02,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:02,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:02,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:03,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:04,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:06,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:06,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:06,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:06,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:07,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:07,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:08,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:09,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:09,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:09,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:09,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:10,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:12,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:12,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:15,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:15,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:15,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:16,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:18,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:18,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:23,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:24,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:24,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:24,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:24,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:25,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:26,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:26,206 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:38:27,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:27,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:28,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:28,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:30,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:31,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:31,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:31,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:31,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:32,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:34,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:34,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:35,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:35,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:35,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:35,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:35,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:38,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:39,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:39,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:39,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:39,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:39,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:40,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:42,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:44,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:44,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:44,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:44,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:44,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:45,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:46,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:48,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:48,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:48,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:48,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:49,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:50,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:50,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:53,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:53,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:53,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:53,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:54,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:56,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:56,211 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:38:56,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:57,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:57,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:57,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:57,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:57,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:00,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:00,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:00,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:00,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:00,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:01,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:01,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:02,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:04,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:04,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:04,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:04,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:05,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:06,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:06,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:09,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:09,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:09,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:09,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:10,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:12,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:12,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:12,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:12,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:12,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:12,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:13,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:14,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:17,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:17,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:17,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:17,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:17,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:18,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:20,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:21,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:21,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:21,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:21,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:22,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:23,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:24,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:24,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:24,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:24,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:24,890 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:25,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:26,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:26,210 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:39:29,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:29,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:29,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:30,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:32,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:33,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:33,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:33,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:33,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:33,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:35,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:36,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:36,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:36,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:36,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:36,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:37,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:38,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:40,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:42,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:42,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:42,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:42,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:43,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:44,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:45,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:45,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:45,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:45,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:46,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:48,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:49,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:49,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:49,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:50,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:51,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:52,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:53,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:53,523 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:53,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:53,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:54,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:56,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:56,216 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:39:57,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:58,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:58,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:58,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:58,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:59,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:00,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:02,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:02,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:02,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:03,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:04,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:06,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:06,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:08,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:08,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:09,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:09,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:09,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:09,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:10,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:12,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:13,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:13,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:13,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:13,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:13,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:14,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:16,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:18,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:18,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:18,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:18,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:18,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:19,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:20,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:22,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:22,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:22,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:22,322 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:23,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:24,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:24,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:26,215 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:40:27,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:27,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:27,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:27,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:28,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:29,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:30,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:31,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:31,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:31,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:31,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:32,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:34,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:35,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:35,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:35,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:35,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:35,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:35,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:36,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:38,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:38,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:38,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:38,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:39,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:40,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:40,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:45,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:45,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:46,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:46,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:46,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:46,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:47,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:48,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:49,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:49,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:49,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:49,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:50,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:51,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:52,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:55,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:55,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:55,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:55,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:56,220 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:40:56,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:57,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:59,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:59,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:59,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:59,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:00,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:02,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:02,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:04,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:04,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:04,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:04,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:04,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:06,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:08,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:09,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:09,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:09,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:09,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:10,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:12,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:13,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:13,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:13,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:13,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:13,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:14,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:16,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:17,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:17,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:17,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:17,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:18,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:18,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:20,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:21,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:21,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:21,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:21,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:22,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:23,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:24,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:26,219 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:41:29,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:30,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:30,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:30,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:30,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:30,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:32,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:34,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:34,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:34,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:34,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:34,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:35,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:36,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:38,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:38,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:38,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:38,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:39,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:39,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:40,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:44,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:45,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:45,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:45,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:45,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:45,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:46,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:49,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:49,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:49,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:49,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:50,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:50,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:52,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:53,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:53,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:53,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:53,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:53,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:54,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:56,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:56,221 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:41:56,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:56,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:56,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:56,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:57,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:58,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:01,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:01,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:01,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:01,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:01,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:02,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:02,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:05,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:05,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:05,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:05,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:05,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:06,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:07,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:08,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:09,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:10,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:12,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:18,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:18,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:18,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:18,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:18,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:18,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:20,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:21,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:21,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:21,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:21,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:23,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:26,223 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:42:26,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:26,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:26,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:26,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:27,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:28,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:29,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:29,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:29,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:29,863 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:29,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:30,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:30,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:32,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:33,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:33,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:33,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:33,590 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:34,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:34,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:36,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:37,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:37,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:37,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:37,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:38,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:39,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:40,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:43,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:43,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:43,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:43,241 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:43,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:46,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:49,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:49,192 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:49,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:49,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:49,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:51,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:52,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:54,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:54,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:54,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:54,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:54,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:56,225 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:42:56,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:56,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:58,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:58,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:00,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:01,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:02,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:02,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:02,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:02,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:04,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:05,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:05,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:05,472 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:05,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:06,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:07,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:08,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:10,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:10,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:10,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:10,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:11,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:12,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:13,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:14,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:14,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:14,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:15,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:16,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:18,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:18,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:18,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:18,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:18,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:19,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:20,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:22,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:22,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:22,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:22,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:23,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:23,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:24,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:26,227 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:43:28,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:28,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:28,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:28,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:28,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:29,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:30,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:30,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:34,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:34,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:34,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:34,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:35,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:36,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:38,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:38,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:38,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:38,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:38,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:40,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:40,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:41,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:41,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:41,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:41,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:42,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:44,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:45,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:45,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:45,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:45,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:45,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:45,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:46,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:48,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:48,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:48,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:48,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:49,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:50,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:50,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:52,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:52,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:52,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:52,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:52,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:54,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:55,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:55,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:55,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:55,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:56,229 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:43:56,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:56,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:58,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:59,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:59,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:59,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:59,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:00,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:01,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:02,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:02,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:02,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:02,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:02,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:03,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:04,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:06,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:06,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:06,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:06,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:06,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:07,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:10,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:10,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:10,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:10,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:11,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:12,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:12,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:15,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:15,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:15,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:16,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:16,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:18,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:18,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:19,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:19,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:19,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:19,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:19,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:22,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:23,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:24,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:24,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:24,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:24,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:24,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:26,234 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:44:26,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:27,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:27,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:27,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:27,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:28,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:28,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:30,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:34,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:36,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:36,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:36,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:36,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:36,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:38,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:40,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:41,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:41,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:41,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:41,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:42,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:45,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:45,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:45,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:45,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:45,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:45,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:46,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:48,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:48,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:48,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:48,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:49,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:50,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:50,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:54,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:54,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:54,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:54,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:54,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:56,233 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:44:56,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:56,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:58,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:58,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:58,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:58,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:58,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:00,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:01,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:06,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:07,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:07,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:07,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:07,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:07,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:08,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:10,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:10,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:10,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:10,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:10,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:12,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:12,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:14,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:14,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:14,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:14,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:14,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:15,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:16,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:17,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:17,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:17,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:17,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:17,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:18,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:20,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:21,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:21,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:21,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:21,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:21,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:22,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:23,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:26,238 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:45:26,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:26,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:26,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:26,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:27,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:28,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:28,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:31,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:31,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:31,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:31,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:32,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:33,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:34,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:37,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:37,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:37,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:38,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:39,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:43,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:43,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:43,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:43,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:44,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:44,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:47,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:47,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:47,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:47,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:47,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:49,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:50,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:51,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:51,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:51,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:51,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:52,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:54,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:54,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:56,238 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:45:57,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:57,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:57,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:57,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:58,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:00,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:00,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:01,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:01,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:01,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:01,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:01,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:02,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:04,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:04,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:04,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:04,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:05,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:05,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:06,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:08,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:08,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:08,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:08,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:08,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:10,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:11,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:12,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:12,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:12,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:12,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:13,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:14,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:15,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:16,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:16,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:16,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:16,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:16,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:17,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:18,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:21,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:21,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:21,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:21,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:21,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:22,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:24,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:25,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:25,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:25,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:25,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:25,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:26,243 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:46:27,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:28,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:29,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:29,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:29,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:29,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:29,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:30,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:32,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:32,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:32,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:32,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:32,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:33,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:34,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:36,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:36,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:36,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:36,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:36,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:37,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:40,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:40,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:40,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:40,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:40,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:42,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:43,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:43,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:43,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:43,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:43,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:44,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:46,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:47,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:47,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:47,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:47,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:47,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:48,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:50,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:51,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:51,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:51,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:51,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:51,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:52,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:54,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:55,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:55,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:55,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:55,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:55,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:56,242 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:46:56,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:58,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:58,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:58,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:58,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:59,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:59,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:00,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:02,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:02,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:02,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:02,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:03,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:04,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:04,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:06,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:06,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:06,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:06,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:08,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:10,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:10,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:10,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:10,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:12,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:14,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:14,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:14,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:14,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:15,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:16,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:20,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:22,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:22,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:22,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:22,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:22,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:24,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:26,247 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:47:26,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:26,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:26,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:26,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:26,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:27,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:28,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:31,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:31,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:31,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:31,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:31,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:32,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:32,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:35,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:35,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:35,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:35,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:36,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:37,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:38,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:40,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:40,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:40,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:40,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:40,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:42,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:43,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:43,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:43,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:43,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:43,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:44,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:46,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:48,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:49,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:49,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:49,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:49,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:49,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:50,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:52,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:52,521 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:52,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:52,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:52,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:54,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:54,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:56,246 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:47:57,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:57,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:57,682 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:57,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:57,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:59,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:00,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:01,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:01,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:01,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:01,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:02,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:04,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:04,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:06,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:06,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:06,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:06,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:06,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:08,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:10,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:10,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:10,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:10,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:10,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:12,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:15,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:16,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:16,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:16,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:16,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:16,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:18,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:21,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:21,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:21,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:21,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:21,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:21,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:22,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:25,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:25,046 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:25,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:25,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:25,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:26,248 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:48:26,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:26,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:28,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:28,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:28,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:28,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:29,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:30,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:31,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:34,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:34,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:34,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:34,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:34,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:36,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:37,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:39,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:39,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:39,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:39,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:39,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:40,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:43,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:43,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:43,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:43,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:43,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:43,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:44,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:47,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:47,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:47,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:47,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:47,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:48,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:48,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:51,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:51,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:51,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:51,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:51,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:54,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:55,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:55,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:55,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:55,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:55,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:56,250 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:48:56,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:58,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:58,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:58,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:58,078 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:58,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:59,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:00,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:02,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:02,465 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:02,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:02,470 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:02,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:04,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:04,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:08,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:08,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:08,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:08,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:08,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:09,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:10,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:12,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:12,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:12,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:12,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:12,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:14,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:15,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:16,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:16,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:18,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:19,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:19,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:19,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:19,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:19,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:20,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:21,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:22,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:22,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:22,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:22,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:22,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:24,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:25,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:25,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:25,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:26,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:26,252 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:49:26,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:26,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:28,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:31,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:34,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:34,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:34,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:34,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:34,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:36,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:37,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:38,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:38,923 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:38,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:38,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:39,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:40,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:41,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:41,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:41,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:41,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:42,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:42,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:44,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:46,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:46,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:46,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:46,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:48,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:48,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:49,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:49,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:49,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:49,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:49,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:50,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:52,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:52,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:52,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:52,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:52,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:53,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:54,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:56,254 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:49:56,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:56,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:56,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:56,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:56,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:58,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:59,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:01,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:01,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:01,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:02,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:05,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:05,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:05,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:05,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:05,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:05,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:06,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:10,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:10,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:10,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:10,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:10,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:11,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:12,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:13,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:13,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:13,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:13,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:13,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:16,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:16,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:19,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:19,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:19,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:19,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:19,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:21,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:22,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:23,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:23,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:23,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:23,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:23,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:24,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:26,256 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:50:26,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:26,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:26,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:26,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:26,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:27,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:28,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:33,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:34,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:34,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:34,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:34,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:36,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:38,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:38,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:38,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:38,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:38,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:39,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:40,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:44,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:47,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:47,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:47,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:47,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:49,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:49,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:51,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:51,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:51,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:51,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:52,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:54,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:55,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:56,261 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:50:56,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:56,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:56,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:56,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:57,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:59,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:59,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:59,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:59,796 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:59,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:00,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:00,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:00,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:03,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:04,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:04,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:04,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:04,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:05,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:05,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:07,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:09,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:09,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:10,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:11,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:11,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:12,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:12,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:12,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:12,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:13,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:15,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:16,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:18,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:18,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:18,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:18,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:19,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:21,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:21,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:23,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:23,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:23,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:24,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:25,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:26,261 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:51:27,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:27,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:27,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:27,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:27,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:27,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:29,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:32,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:32,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:32,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:32,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:32,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:33,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:35,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:37,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:40,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:40,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:40,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:40,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:41,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:42,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:43,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:44,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:44,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:44,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:44,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:45,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:47,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:48,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:48,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:48,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:48,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:48,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:49,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:51,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:52,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:52,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:52,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:52,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:53,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:53,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:55,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:56,268 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:51:56,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:56,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:56,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:56,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:57,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:58,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:59,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:00,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:01,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:01,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:01,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:01,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:01,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:03,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:04,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:04,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:04,515 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:04,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:04,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:05,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:07,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:08,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:08,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:08,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:08,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:09,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:11,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:11,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:11,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:11,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:12,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:12,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:15,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:15,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:17,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:17,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:17,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:17,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:18,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:19,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:20,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:20,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:20,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:20,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:20,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:21,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:23,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:23,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:23,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:23,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:23,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:24,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:25,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:25,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:26,265 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:52:27,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:27,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:27,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:27,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:28,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:29,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:30,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:30,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:30,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:30,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:31,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:31,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:33,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:35,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:35,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:35,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:35,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:36,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:37,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:37,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:41,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:41,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:41,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:41,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:42,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:42,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:43,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:46,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:46,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:46,863 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:46,868 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:47,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:49,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:51,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:51,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:51,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:51,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:52,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:53,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:53,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:55,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:55,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:55,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:55,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:56,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:56,270 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:52:57,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:58,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:01,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:01,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:01,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:01,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:02,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:03,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:03,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:06,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:06,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:06,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:06,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:07,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:09,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:09,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:14,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:14,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:14,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:15,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:17,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:19,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:20,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:20,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:20,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:20,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:21,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:23,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:24,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:26,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:26,269 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:53:27,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:29,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:29,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:29,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:29,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:29,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:30,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:30,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:31,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:33,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:33,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:33,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:33,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:34,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:35,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:36,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:36,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:36,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:36,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:36,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:37,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:39,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:41,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:41,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:41,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:41,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:42,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:42,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:44,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:44,792 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:44,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:44,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:45,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:45,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:47,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:47,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:48,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:48,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:48,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:48,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:48,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:51,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:51,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:51,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:51,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:53,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:53,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:56,271 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:53:58,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:58,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:58,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:58,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:58,781 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:59,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:01,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:02,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:02,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:02,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:02,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:03,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:03,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:05,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:05,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:05,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:05,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:05,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:06,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:07,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:08,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:09,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:09,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:09,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:09,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:09,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:11,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:12,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:12,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:12,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:12,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:13,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:14,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:15,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:17,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:17,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:17,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:17,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:18,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:19,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:19,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:20,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:20,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:20,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:20,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:21,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:23,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:24,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:26,273 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:54:30,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:30,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:30,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:30,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:30,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:30,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:33,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:33,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:33,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:33,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:33,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:34,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:35,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:35,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:36,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:36,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:36,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:36,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:37,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:39,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:39,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:39,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:39,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:39,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:40,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:40,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:45,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:46,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:47,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:47,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:47,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:47,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:48,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:51,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:52,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:56,276 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:54:56,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:56,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:56,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:56,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:57,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:57,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:59,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:59,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:59,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:59,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:59,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:00,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:01,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:02,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:03,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:03,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:03,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:03,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:04,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:05,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:07,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:07,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:07,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:07,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:07,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:08,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:09,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:11,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:11,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:11,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:11,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:12,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:13,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:13,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:17,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:17,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:17,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:17,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:18,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:19,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:19,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:21,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:21,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:21,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:21,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:21,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:23,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:24,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:24,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:24,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:24,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:24,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:25,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:26,278 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:55:27,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:28,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:28,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:28,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:28,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:29,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:29,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:31,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:33,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:33,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:33,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:33,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:33,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:35,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:35,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:36,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:36,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:37,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:39,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:40,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:45,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:45,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:45,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:45,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:45,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:46,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:47,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:48,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:48,542 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:48,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:48,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:49,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:51,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:51,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:54,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:54,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:54,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:54,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:54,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:56,280 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:55:57,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:57,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:57,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:57,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:57,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:58,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:59,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:01,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:01,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:02,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:02,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:03,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:05,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:05,237 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:05,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:05,242 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:07,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:08,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:10,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:10,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:10,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:10,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:11,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:13,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:13,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:18,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:19,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:19,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:19,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:19,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:19,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:21,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:24,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:24,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:24,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:24,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:24,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:24,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:26,285 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:56:27,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:27,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:27,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:27,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:27,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:28,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:29,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:29,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:30,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:31,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:31,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:31,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:31,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:32,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:33,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:34,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:34,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:34,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:34,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:35,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:35,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:37,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:38,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:38,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:38,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:38,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:38,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:39,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:40,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:42,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:42,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:42,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:42,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:43,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:45,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:45,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:45,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:45,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:45,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:45,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:46,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:47,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:50,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:50,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:50,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:50,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:50,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:51,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:53,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:56,284 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:56:56,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:56,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:56,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:56,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:57,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:59,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:01,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:02,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:02,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:02,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:02,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:05,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:06,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:06,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:06,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:06,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:06,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:07,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:07,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:09,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:09,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:09,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:09,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:09,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:11,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:12,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:12,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:12,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:12,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:12,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:12,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:15,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:16,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:16,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:16,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:16,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:17,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:17,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:19,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:21,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:21,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:21,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:21,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:21,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:23,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:23,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:25,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:25,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:25,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:26,289 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:57:27,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:28,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:30,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:33,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:33,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:33,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:33,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:33,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:34,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:35,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:38,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:42,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:42,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:42,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:42,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:42,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:43,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:44,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:47,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:47,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:47,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:47,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:47,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:49,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:49,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:50,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:50,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:50,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:50,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:51,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:53,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:53,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:53,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:53,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:53,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:54,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:54,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:55,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:56,288 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:57:56,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:56,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:56,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:56,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:57,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:59,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:59,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:00,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:00,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:00,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:00,958 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:01,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:03,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:04,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:09,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:09,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:09,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:09,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:10,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:11,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:12,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:12,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:12,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:12,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:13,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:15,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:15,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:20,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:20,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:20,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:20,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:20,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:21,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:23,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:24,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:24,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:24,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:24,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:25,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:25,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:26,293 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:58:27,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:31,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:33,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:33,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:33,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:33,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:34,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:35,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:36,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:36,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:36,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:36,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:36,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:37,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:39,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:40,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:40,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:40,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:40,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:40,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:41,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:42,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:44,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:44,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:44,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:44,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:44,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:45,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:47,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:47,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:47,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:47,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:47,914 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:48,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:49,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:51,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:51,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:51,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:53,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:53,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:56,292 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:58:59,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:59,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:59,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:59,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:59,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:00,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:01,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:04,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:05,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:05,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:05,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:05,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:05,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:07,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:08,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:08,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:08,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:08,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:08,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:09,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:10,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:12,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:12,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:12,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:12,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:12,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:15,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:15,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:15,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:15,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:15,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:15,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:16,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:17,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:18,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:18,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:18,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:18,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:19,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:20,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:21,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:22,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:22,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:22,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:22,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:22,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:25,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:25,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:25,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:25,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:25,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:26,294 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:59:26,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:26,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:27,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:31,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:32,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:32,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:32,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:33,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:35,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:36,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:36,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:36,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:36,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:36,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:37,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:39,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:40,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:40,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:40,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:40,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:40,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:42,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:43,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:43,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:44,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:45,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:46,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:46,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:46,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:46,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:47,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:47,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:49,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:50,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:50,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:50,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:50,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:50,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:53,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:53,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:54,794 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:54,794 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:54,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:55,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:56,296 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:59:57,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:59,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:59,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:59,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:59,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:59,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:59,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:01,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:03,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:03,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:03,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:03,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:03,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:04,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:05,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:07,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:07,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:07,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:07,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:08,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:09,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:09,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:12,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:12,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:12,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:12,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:12,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:13,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:15,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:15,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:15,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:15,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:15,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:15,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:16,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:17,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:20,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:24,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:24,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:24,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:24,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:24,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:25,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:26,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:26,298 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:00:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:31,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:31,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:31,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:31,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:31,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:31,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:33,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:35,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:35,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:35,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:35,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:35,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:37,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:37,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:38,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:38,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:38,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:38,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:38,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:39,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:42,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:43,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:43,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:43,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:43,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:44,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:45,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:47,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:47,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:47,523 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:47,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:47,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:49,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:51,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:51,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:51,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:51,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:51,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:53,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:53,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:54,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:54,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:54,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:54,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:54,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:56,301 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:00:57,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:57,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:57,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:57,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:57,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:58,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:58,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:59,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:02,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:02,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:02,086 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:02,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:02,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:03,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:04,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:05,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:05,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:05,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:05,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:05,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:07,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:09,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:09,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:09,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:09,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:09,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:10,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:11,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:14,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:14,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:14,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:14,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:14,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:15,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:15,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:17,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:17,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:17,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:17,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:17,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:19,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:20,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:23,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:23,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:23,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:23,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:23,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:25,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:26,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:26,303 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:01:28,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:28,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:28,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:28,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:29,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:31,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:31,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:36,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:37,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:37,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:37,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:37,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:37,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:39,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:42,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:45,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:45,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:45,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:46,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:47,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:50,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:50,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:50,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:50,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:51,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:53,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:53,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:54,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:54,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:54,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:54,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:54,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:56,308 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:01:57,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:58,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:58,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:58,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:58,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:59,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:59,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:01,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:02,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:02,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:02,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:02,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:03,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:05,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:05,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:07,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:07,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:07,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:08,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:08,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:09,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:11,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:12,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:12,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:12,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:12,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:12,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:13,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:16,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:18,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:18,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:18,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:18,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:18,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:21,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:21,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:22,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:22,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:22,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:22,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:22,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:23,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:26,307 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:02:27,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:27,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:27,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:27,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:27,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:27,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:29,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:30,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:30,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:30,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:30,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:30,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:32,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:33,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:33,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:35,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:37,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:39,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:39,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:39,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:39,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:39,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:41,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:42,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:42,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:42,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:42,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:42,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:43,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:45,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:48,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:51,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:51,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:51,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:51,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:51,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:53,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:54,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:55,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:55,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:55,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:55,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:56,312 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:02:57,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:00,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:00,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:00,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:00,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:00,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:00,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:01,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:03,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:03,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:03,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:03,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:04,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:05,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:05,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:07,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:07,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:07,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:07,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:08,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:09,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:10,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:14,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:14,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:14,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:14,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:14,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:15,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:16,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:19,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:19,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:19,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:19,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:19,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:21,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:21,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:25,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:25,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:25,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:25,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:26,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:03:26,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:27,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:27,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:29,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:29,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:29,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:29,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:30,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:32,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:33,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:33,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:33,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:33,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:33,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:35,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:36,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:36,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:36,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:36,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:37,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:37,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:39,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:42,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:45,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:45,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:45,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:45,577 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:45,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:47,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:48,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:48,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:48,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:48,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:48,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:51,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:52,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:52,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:52,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:52,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:52,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:53,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:55,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:56,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:56,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:56,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:56,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:56,316 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:03:56,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:57,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:59,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:01,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:01,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:03,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:04,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:04,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:04,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:04,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:04,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:04,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:08,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:08,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:08,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:08,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:08,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:09,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:10,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:11,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:11,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:11,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:11,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:12,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:13,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:15,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:17,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:17,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:17,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:17,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:18,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:19,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:20,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:22,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:22,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:22,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:22,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:23,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:25,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:25,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:26,315 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:04:26,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:26,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:26,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:26,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:27,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:29,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:31,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:31,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:31,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:31,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:31,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:31,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:33,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:35,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:35,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:35,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:35,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:35,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:37,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:37,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:38,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:38,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:38,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:38,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:39,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:42,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:42,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:42,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:42,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:42,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:43,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:45,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:46,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:46,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:46,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:46,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:46,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:47,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:48,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:51,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:51,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:51,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:51,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:51,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:53,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:53,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:56,317 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:04:56,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:56,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:57,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:58,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:00,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:00,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:00,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:00,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:00,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:00,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:01,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:03,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:03,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:03,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:03,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:03,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:04,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:05,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:07,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:07,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:07,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:07,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:07,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:09,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:09,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:10,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:10,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:10,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:10,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:11,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:13,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:14,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:14,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:14,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:14,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:14,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:15,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:20,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:20,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:20,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:20,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:20,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:20,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:21,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:24,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:24,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:24,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:24,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:24,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:25,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:25,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:26,319 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:05:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:31,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:33,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:33,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:33,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:33,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:33,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:35,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:36,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:36,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:36,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:36,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:36,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:36,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:37,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:40,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:40,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:40,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:40,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:40,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:41,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:42,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:43,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:43,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:43,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:43,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:43,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:45,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:47,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:48,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:48,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:48,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:48,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:48,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:49,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:52,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:53,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:53,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:53,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:53,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:53,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:55,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:56,321 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:05:57,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:57,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:57,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:57,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:57,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:58,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:59,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:00,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:02,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:02,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:02,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:02,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:02,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:03,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:04,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:06,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:06,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:06,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:06,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:06,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:09,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:09,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:10,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:10,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:10,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:10,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:10,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:11,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:15,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:15,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:15,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:15,571 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:15,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:17,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:19,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:19,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:19,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:19,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:19,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:21,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:21,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:22,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:22,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:22,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:22,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:22,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:23,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:26,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:26,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:26,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:26,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:26,324 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:06:26,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:26,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:27,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:30,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:30,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:30,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:30,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:30,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:31,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:31,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:35,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:35,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:35,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:35,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:35,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:37,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:37,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:38,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:38,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:38,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:38,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:39,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:42,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:42,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:42,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:42,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:42,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:43,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:46,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:46,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:46,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:46,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:46,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:47,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:48,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:53,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:54,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:54,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:54,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:54,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:54,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:55,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:56,326 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:06:57,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:57,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:58,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:59,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:00,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:00,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:00,916 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:00,921 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:00,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:03,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:03,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:04,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:04,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:04,071 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:04,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:04,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:05,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:07,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:07,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:07,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:07,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:09,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:09,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:14,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:16,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:16,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:16,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:16,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:17,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:18,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:20,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:20,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:20,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:20,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:20,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:23,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:23,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:23,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:23,387 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:24,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:25,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:26,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:26,331 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:07:26,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:26,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:26,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:26,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:27,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:28,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:29,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:29,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:29,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:29,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:30,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:30,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:32,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:33,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:33,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:33,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:33,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:34,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:35,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:36,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:37,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:37,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:38,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:40,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:41,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:41,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:41,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:41,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:42,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:44,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:45,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:46,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:46,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:46,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:46,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:46,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:47,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:50,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:51,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:55,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:55,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:55,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:55,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:56,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:56,330 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:07:57,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:58,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:59,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:59,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:59,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:59,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:00,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:02,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:02,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:03,004 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:03,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:06,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:06,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:06,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:06,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:06,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:07,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:07,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:08,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:12,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:15,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:15,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:15,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:15,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:16,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:18,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:18,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:20,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:20,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:20,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:20,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:21,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:22,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:24,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:25,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:25,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:25,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:25,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:26,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:26,335 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:08:28,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:28,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:28,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:28,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:28,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:29,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:29,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:30,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:31,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:31,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:31,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:31,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:32,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:34,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:34,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:35,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:35,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:35,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:35,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:36,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:38,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:39,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:39,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:39,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:39,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:40,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:40,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:42,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:44,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:44,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:44,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:44,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:45,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:45,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:46,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:49,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:49,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:49,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:49,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:50,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:50,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:52,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:55,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:56,334 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:08:57,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:57,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:57,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:57,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:58,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:00,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:00,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:01,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:01,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:01,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:01,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:01,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:04,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:06,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:07,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:07,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:07,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:07,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:07,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:10,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:10,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:10,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:10,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:10,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:11,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:11,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:12,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:17,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:18,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:18,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:18,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:18,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:19,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:22,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:23,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:24,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:24,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:24,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:24,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:25,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:26,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:26,338 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:09:27,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:27,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:27,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:27,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:28,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:28,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:30,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:32,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:32,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:32,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:32,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:33,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:34,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:34,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:39,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:41,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:41,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:41,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:41,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:41,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:44,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:44,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:44,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:44,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:44,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:45,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:45,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:46,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:47,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:47,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:47,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:47,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:48,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:50,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:50,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:50,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:50,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:50,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:51,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:52,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:55,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:55,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:55,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:55,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:55,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,338 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:09:58,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:00,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:00,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:00,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:00,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:01,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:01,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:03,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:03,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:03,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:03,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:04,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:06,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:07,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:07,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:07,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:07,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:07,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:08,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:10,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:12,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:16,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:16,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:16,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:16,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:17,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:17,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:18,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:22,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:25,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:25,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:26,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:26,340 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:10:28,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:28,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:28,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:28,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:28,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:28,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:29,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:30,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:32,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:32,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:32,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:32,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:33,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:34,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:36,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:36,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:36,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:36,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:37,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:38,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:39,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:39,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:39,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:39,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:39,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:40,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:42,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:44,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:44,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:44,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:44,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:44,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:45,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:46,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:47,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:47,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:47,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:47,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:48,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:50,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:50,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:50,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:50,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:50,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:50,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:51,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:52,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:55,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:56,342 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:10:59,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:59,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:59,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:59,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:59,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:00,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:02,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:02,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:02,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:02,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:03,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:04,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:06,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:08,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:08,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:08,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:09,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:10,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:11,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:13,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:13,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:13,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:13,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:14,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:16,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:17,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:17,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:17,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:17,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:18,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:21,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:21,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:21,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:21,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:21,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:22,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:24,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:26,344 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:11:27,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:27,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:27,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:27,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:27,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:28,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:30,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:32,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:32,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:32,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:32,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:32,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:33,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:34,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:37,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:37,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:37,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:37,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:37,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:38,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:40,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:42,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:43,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:43,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:43,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:43,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:43,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:44,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:47,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:47,515 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:47,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:47,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:47,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:48,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:50,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:51,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:51,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:51,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:51,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:52,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:52,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:54,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:54,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:54,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:54,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:54,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:56,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:56,346 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:11:58,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:58,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:58,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:58,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:58,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:59,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:00,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:03,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:03,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:03,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:03,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:03,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:04,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:06,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:07,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:07,711 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:07,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:07,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:08,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:08,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:10,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:11,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:11,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:11,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:11,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:11,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:14,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:14,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:15,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:15,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:15,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:15,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:16,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:18,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:19,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:19,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:19,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:19,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:19,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:20,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:22,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:24,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:24,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:24,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:24,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:24,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:25,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:26,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:26,348 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:12:29,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:29,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:29,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:29,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:30,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:30,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:32,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:33,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:33,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:33,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:33,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:33,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:34,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:36,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:38,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:38,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:38,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:38,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:39,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:40,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:41,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:41,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:41,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:41,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:41,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:42,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:44,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:45,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:45,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:45,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:45,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:46,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:46,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:48,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:50,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:50,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:50,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:50,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:50,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:52,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:52,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:55,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:55,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:55,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:55,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:56,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:56,354 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:12:57,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:58,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:58,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:58,819 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:58,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:58,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:59,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:00,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:02,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:05,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:05,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:07,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:08,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:10,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:10,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:10,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:10,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:11,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:12,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:13,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:13,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:13,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:13,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:13,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:14,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:16,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:17,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:17,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:17,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:17,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:17,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:18,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:19,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:20,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:20,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:20,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:20,417 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:21,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:22,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:23,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:23,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:23,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:23,943 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:24,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:24,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:26,353 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:13:26,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:28,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:28,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:28,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:28,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:29,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:30,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:30,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:30,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:32,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:32,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:32,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:32,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:33,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:34,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:36,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:37,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:37,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:37,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:37,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:38,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:40,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:41,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:42,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:42,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:42,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:42,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:43,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:44,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:45,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:47,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:48,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:48,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:48,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:48,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:49,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:50,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:52,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:52,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:52,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:52,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:52,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:53,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:54,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:56,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:56,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:56,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:56,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:56,358 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:13:56,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:58,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:58,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:59,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:59,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:00,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:02,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:03,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:04,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:04,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:04,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:04,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:05,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:06,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:08,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:08,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:08,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:08,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:08,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:09,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:10,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:12,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:12,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:12,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:12,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:12,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:14,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:14,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:16,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:16,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:16,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:16,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:17,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:18,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:19,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:22,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:22,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:22,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:22,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:23,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:24,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:24,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:26,357 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:14:28,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:28,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:28,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:28,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:28,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:30,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:30,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:30,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:31,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:31,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:31,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:31,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:32,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:34,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:34,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:34,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:34,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:34,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:35,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:35,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:36,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:40,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:42,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:42,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:42,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:42,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:42,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:44,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:45,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:45,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:49,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:49,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:49,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:49,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:50,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:50,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:52,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:54,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:54,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:54,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:54,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:54,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:56,362 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:14:56,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:56,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:58,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:58,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:58,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:58,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:59,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:00,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:01,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:04,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:04,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:04,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:04,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:04,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:06,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:07,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:07,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:07,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:07,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:07,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:08,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:10,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:11,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:11,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:11,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:11,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:12,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:12,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:14,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:15,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:15,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:15,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:15,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:16,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:18,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:18,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:18,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:18,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:18,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:18,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:19,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:20,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:23,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:24,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:24,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:24,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:24,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:24,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:26,361 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:15:26,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:27,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:27,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:27,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:27,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:28,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:29,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:30,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:32,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:32,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:32,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:32,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:32,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:34,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:35,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:35,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:35,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:35,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:38,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:40,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:40,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:40,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:40,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:40,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:40,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:42,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:45,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:48,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:48,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:48,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:48,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:49,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:50,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:50,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:51,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:51,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:51,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:51,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:52,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:54,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:55,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:55,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:55,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:55,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:55,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:56,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:56,363 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:15:56,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:00,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:00,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:00,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:00,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:00,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:00,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:01,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:02,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:06,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:08,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:08,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:08,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:08,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:09,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:10,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:11,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:15,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:17,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:18,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:18,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:18,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:18,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:19,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:20,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:22,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:22,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:22,136 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:22,141 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:22,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:23,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:24,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:26,365 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:16:28,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:28,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:28,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:28,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:28,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:29,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:30,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:31,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:31,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:31,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:31,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:32,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:34,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:39,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:40,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:40,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:40,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:40,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:40,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:42,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:45,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:46,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:46,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:46,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:46,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:47,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:48,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:50,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:52,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:52,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:52,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:52,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:53,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:54,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:56,367 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:16:56,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:56,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:56,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:56,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:57,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:58,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:00,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:01,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:02,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:02,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:02,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:02,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:02,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:04,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:05,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:05,164 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:05,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:05,169 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:05,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:06,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:07,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:08,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:08,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:08,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:08,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:09,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:10,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:12,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:12,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:12,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:12,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:12,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:12,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:14,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:15,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:16,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:16,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:16,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:16,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:17,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:22,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:24,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:24,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:24,043 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:24,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:24,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:26,373 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:17:26,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:27,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:27,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:27,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:27,482 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:27,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:27,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:28,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:32,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:32,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:32,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:32,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:32,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:33,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:34,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:37,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:37,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:37,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:37,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:37,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:38,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:38,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:41,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:41,651 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:41,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:41,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:42,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:43,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:44,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:45,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:45,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:45,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:46,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:49,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:49,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:49,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:49,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:49,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:50,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:52,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:54,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:54,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:54,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:54,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:54,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:55,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:56,372 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:17:56,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:58,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:58,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:58,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:58,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:58,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:00,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:00,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:00,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:02,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:02,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:02,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:02,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:02,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:04,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:06,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:06,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,335 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:06,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:06,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:08,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:10,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:10,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:10,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:10,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:10,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:12,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:12,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:13,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:13,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:13,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:13,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:14,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:16,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:17,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:17,460 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:17,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:17,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:17,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:18,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:22,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:23,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:23,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:23,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:23,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:23,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:24,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:26,376 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:18:26,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:26,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:26,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:26,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:27,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:27,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:28,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:32,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:32,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:32,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:32,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:32,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:33,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:34,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:35,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:35,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:35,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:36,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:38,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:38,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:39,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:39,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:39,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:39,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:40,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:42,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:44,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:45,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:45,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:45,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:45,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:45,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:46,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:48,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:48,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:48,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:48,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:49,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:49,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:50,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:52,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:52,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:52,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:52,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:52,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:54,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:55,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:56,376 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:18:56,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:56,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:56,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:56,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:56,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:58,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:59,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:59,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:59,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:59,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:00,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:00,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:00,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:02,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:03,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:03,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:03,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:03,534 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:03,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:04,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:06,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:06,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:06,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:06,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:06,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:07,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:08,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:09,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:09,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:09,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:10,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:10,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:12,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:12,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:13,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:13,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:13,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:13,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:13,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:14,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:17,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:18,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:18,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:18,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:18,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:18,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:20,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:21,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:21,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:21,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:21,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:21,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:22,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:24,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:24,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:24,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:24,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:24,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:25,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:26,381 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:19:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:28,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:31,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:31,242 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:31,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:31,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:31,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:32,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:34,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:34,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:34,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:34,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:34,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:34,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:36,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:39,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:42,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:42,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:42,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:42,935 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:43,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:44,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:44,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:45,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:45,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:45,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:45,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:46,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:48,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:49,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:49,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:49,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:50,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:50,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:52,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:53,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:53,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:53,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:53,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:53,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:54,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:56,380 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:19:56,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:57,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:57,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:57,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:57,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:57,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:58,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:00,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:00,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:00,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:01,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:01,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:02,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:04,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:04,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:04,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:04,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:04,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:06,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:07,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:07,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:07,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:07,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:07,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:07,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:08,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:12,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:12,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:12,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:12,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:12,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:13,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:14,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:16,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:16,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:16,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:16,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:16,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:17,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:20,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:20,382 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:20,382 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:20,387 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:20,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:22,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:23,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:23,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:23,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:23,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:23,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:26,385 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:20:26,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:27,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:27,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:27,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:27,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:27,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:28,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:29,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:30,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:31,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:31,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:31,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:31,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:31,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:32,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:35,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:35,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:35,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:35,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:35,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:35,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:36,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:38,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:38,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:38,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:38,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:39,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:40,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:40,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:42,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:42,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:42,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:42,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:42,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:44,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:45,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:46,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:46,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:46,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:46,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:46,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:46,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:48,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:51,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:51,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:51,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:51,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:51,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:51,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:52,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:54,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:54,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:54,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:54,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:54,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:56,384 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:20:56,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:56,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:58,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:58,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:58,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:58,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:58,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:00,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:00,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:01,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:01,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:02,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:04,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:06,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:08,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:08,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:08,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:08,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:09,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:10,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:11,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:12,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:12,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:12,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:12,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:12,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:14,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:15,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:15,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:19,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:24,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:26,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:27,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:31,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:34,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:35,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:36,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:36,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,902 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:21:36,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:37,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:38,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:38,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:38,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:38,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:38,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:38,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:40,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:40,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:40,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:40,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:40,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:40,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:41,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:41,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:42,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:46,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:48,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:48,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:48,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:48,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:48,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:50,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:50,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:52,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:53,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:53,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:53,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:53,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:53,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:54,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:55,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:56,388 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:21:57,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:57,898 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:57,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:57,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:57,903 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:58,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:00,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:01,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:01,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:01,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:01,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:01,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:02,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:03,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:05,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:05,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:05,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:05,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:05,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:05,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:06,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:09,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:10,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:10,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:10,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:10,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:10,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:10,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:12,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:13,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:13,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:13,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:13,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:13,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:14,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:14,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:19,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:19,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:19,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:19,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:19,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:20,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:20,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:22,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:24,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:24,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:24,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:24,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:25,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:25,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:26,393 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:22:26,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:29,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:29,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:29,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:29,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:29,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:30,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:31,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:32,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:32,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:32,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:33,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:33,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:34,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:35,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:37,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:39,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:39,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:39,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:39,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:39,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:40,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:40,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:42,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:42,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:42,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:42,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:42,775 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:42,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:44,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:47,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:47,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:47,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:47,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:47,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:48,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:48,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:50,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:50,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:50,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:50,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:50,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:50,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:52,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:53,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:55,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:55,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:55,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:55,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:55,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:55,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:56,392 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:22:56,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:59,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:59,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:59,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:59,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:59,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:59,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:00,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:04,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:04,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:04,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:04,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:05,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:05,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:06,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:08,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:08,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:08,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:08,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:08,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:10,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:10,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:11,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:11,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:11,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:11,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:11,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:11,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:12,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:16,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:16,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:16,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:16,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:16,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:16,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:18,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:19,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:19,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:19,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:19,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:19,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:20,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:20,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:21,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:23,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:23,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:23,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:23,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:23,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:24,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:25,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:26,394 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:23:26,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:26,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:26,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:26,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:26,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:27,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:28,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:30,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:30,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:30,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:30,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:32,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:32,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:34,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:34,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:34,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:34,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:34,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:35,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:36,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:37,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:37,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:37,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:37,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:37,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:38,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:38,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:40,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:41,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:41,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:41,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:41,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:41,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:42,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:44,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:48,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:48,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:48,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:49,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:50,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:50,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:52,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:52,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:52,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:52,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:52,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:54,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:54,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:55,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:56,396 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:23:56,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:56,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:56,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:56,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:56,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:58,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:59,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:59,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:59,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:59,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:59,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:59,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:00,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:03,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:03,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:03,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:03,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:03,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:04,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:05,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:05,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:07,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:07,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:07,100 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:07,105 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:07,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:08,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:10,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:10,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:10,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:10,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:10,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:10,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:11,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:13,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:14,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:14,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:14,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:14,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:15,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:15,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:17,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:17,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:17,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:17,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:17,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:18,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:19,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:20,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:20,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:22,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:22,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:22,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:22,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:23,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:25,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:25,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:26,399 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:24:26,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:26,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:26,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:26,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:27,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:29,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:31,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:31,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:31,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:31,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:31,676 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:32,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:33,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:35,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:35,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:35,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:35,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:35,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:36,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:37,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:37,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:39,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:39,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:39,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:39,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:40,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:40,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:41,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:42,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:47,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:48,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:48,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:48,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:48,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:48,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:50,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:51,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:53,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:54,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:54,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:54,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:54,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:54,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:55,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:55,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:56,401 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:24:58,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:58,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:58,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:58,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:58,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:59,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:01,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:01,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:01,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:01,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:01,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:02,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:03,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:03,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:05,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:06,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:06,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:06,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:06,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:07,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:09,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:09,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:10,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:10,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:10,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:10,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:10,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:11,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:13,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:14,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:15,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:15,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:15,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:15,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:16,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:17,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:20,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:20,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:21,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:21,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:21,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:21,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:22,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:23,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:25,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:26,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:26,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:26,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:26,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:26,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:26,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:26,403 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:25:27,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:29,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:29,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:29,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:29,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:30,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:31,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:31,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:34,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:34,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:34,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:34,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:35,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:37,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:37,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:38,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:38,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:38,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:38,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:39,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:40,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:41,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:41,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:41,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:41,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:41,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:42,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:42,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:43,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:47,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:47,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:47,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:47,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:47,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:48,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:49,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:50,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:52,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:52,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:52,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:52,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:52,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:53,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:55,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:55,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:56,405 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:25:58,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:00,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:00,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:00,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:00,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:01,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:03,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:03,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:05,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:08,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:08,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:08,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:08,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:08,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:09,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:10,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:11,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:13,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:14,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:14,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:14,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:14,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:14,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:15,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:19,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:20,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:21,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:21,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:21,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:21,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:22,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:23,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:24,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:24,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:24,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:24,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:24,812 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:25,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:25,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:26,407 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:26:27,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:28,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:28,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:28,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:28,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:29,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:29,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:31,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:32,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:32,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:32,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:32,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:32,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:35,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:35,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:35,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:35,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:35,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:35,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:35,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:36,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:37,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:40,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:40,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:40,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:40,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:40,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:40,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:41,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:41,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:43,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:43,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:43,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:43,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:45,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:46,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:47,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:47,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:47,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:47,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:48,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:49,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:50,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:52,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:52,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:52,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:52,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:52,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:53,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:55,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:55,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:56,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:56,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:56,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:56,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:56,413 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:26:57,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:57,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:59,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:59,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:59,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:59,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:59,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:01,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:02,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:03,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:03,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:03,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:03,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:04,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:05,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:05,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:07,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:07,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:07,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:07,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:08,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:08,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:09,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:10,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:11,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:11,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:11,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:11,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:11,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:13,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:14,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:14,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:14,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:15,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:17,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:19,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:20,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:23,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:23,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:23,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:23,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:23,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:25,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:25,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:25,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:26,411 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:27:26,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:26,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:26,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:26,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:27,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:29,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:30,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:32,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:32,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:32,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:32,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:32,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:33,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:35,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:35,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:35,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:35,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:35,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:35,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:36,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:37,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:40,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:40,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:40,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:40,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:41,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:41,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:43,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:43,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:43,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:43,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:43,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:45,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:46,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:47,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:47,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:47,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:47,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:47,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:49,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:50,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:52,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:52,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:52,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:52,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:52,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:53,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:55,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:55,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:56,416 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:27:56,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:56,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:56,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:57,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:57,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:58,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:59,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:00,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:00,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:00,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:00,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:00,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:01,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:03,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:03,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:03,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:03,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:03,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:04,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:05,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:05,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:07,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:07,408 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:07,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:07,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:08,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:08,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:09,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:10,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:11,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:11,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:11,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:11,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:11,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:13,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:14,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:14,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:14,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:14,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:14,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:15,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:17,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:19,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:20,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:20,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:20,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:20,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:20,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:21,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:23,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:24,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:24,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:24,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:24,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:24,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:25,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:25,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:25,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:26,415 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:28:29,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:29,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:29,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:29,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:29,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:31,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:33,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:33,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:33,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:33,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:34,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:35,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:35,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:37,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:37,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:37,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:37,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:37,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:39,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:40,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:40,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:40,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:40,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:40,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:41,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:41,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:43,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:45,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:45,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:45,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:45,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:46,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:46,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:47,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:49,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:49,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:49,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:49,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:49,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:50,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:51,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:52,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:52,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:52,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:52,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:52,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:53,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:55,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:55,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:56,420 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:28:57,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:58,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:58,001 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:58,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:58,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:58,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:59,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:03,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:03,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:03,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:03,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:03,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:03,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:05,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:05,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:06,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:06,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:06,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:06,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:07,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:08,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:09,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:10,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:10,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:10,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:10,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:10,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:11,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:13,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:13,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:14,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:14,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:14,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:14,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:15,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:17,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:18,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:19,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:19,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:19,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:19,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:20,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:20,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:21,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:24,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:25,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:25,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:25,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:25,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:25,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:26,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:26,420 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:29:27,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:29,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:29,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:29,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:29,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:30,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:30,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:31,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:34,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:34,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:35,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:35,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:37,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:37,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:37,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:37,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:37,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:38,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:39,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:40,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:40,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:41,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:41,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:41,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:41,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:41,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:43,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:46,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:46,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:46,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:46,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:46,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:47,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:49,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:50,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:51,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:55,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:55,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:55,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:55,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:55,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:55,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:56,422 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:29:57,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:57,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:59,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:59,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:59,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:59,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:00,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:01,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:02,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:02,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:02,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:02,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:02,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:03,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:05,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:05,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:05,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:05,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:05,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:05,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:05,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:07,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:08,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:09,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:09,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:09,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:09,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:10,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:10,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:11,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:13,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:13,242 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:13,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:13,247 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:13,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:14,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:16,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:16,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:16,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:16,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:16,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:19,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:19,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:20,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:20,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:20,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:20,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:20,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:20,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:21,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:25,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:25,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:25,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:25,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:25,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:25,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:25,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:26,424 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:30:27,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:29,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:29,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:30,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:30,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:31,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:33,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:33,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:33,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:33,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:33,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:35,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:35,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:36,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:36,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:36,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:36,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:36,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:37,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:39,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:40,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:40,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:40,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:40,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:40,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:40,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:41,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:42,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:46,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:46,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:46,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:46,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:46,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:47,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:47,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:50,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:52,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:52,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:52,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:52,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:52,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:53,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:55,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:55,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:56,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:56,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:56,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:56,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:56,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:56,426 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:30:57,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:57,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:00,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:00,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:00,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:00,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:01,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:02,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:03,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:04,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:04,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:04,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:04,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:05,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:05,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:07,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:08,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:08,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:08,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:08,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:08,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:08,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:09,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:10,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:11,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:11,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:11,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:11,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:12,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:13,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:13,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:14,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:14,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:14,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:14,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:15,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:17,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:19,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:19,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:19,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:19,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:19,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:19,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:20,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:21,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:22,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:22,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:22,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:22,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:23,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:24,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:25,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:25,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:26,428 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:31:27,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:27,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:27,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:27,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:29,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:30,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:35,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:35,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:35,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:35,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:35,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:35,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:36,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:37,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:40,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:40,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:42,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:42,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:42,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:42,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:43,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:45,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:46,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:46,783 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:46,783 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:46,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:46,788 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:47,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:49,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:51,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:51,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:53,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:53,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:53,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:53,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:54,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:55,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:56,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:56,430 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:31:57,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:58,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:58,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:58,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:58,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:59,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:01,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:02,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:03,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:03,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:03,311 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:03,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:03,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:05,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:06,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:06,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:06,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:06,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:06,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:07,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:07,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:09,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:11,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:11,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:11,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:11,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:11,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:12,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:13,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:13,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:18,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:20,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:20,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:20,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:20,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:20,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:21,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:21,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:23,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:23,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:23,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:23,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:23,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:24,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:25,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:26,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:26,435 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:32:27,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:27,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:27,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:27,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:27,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:29,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:29,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:32,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:32,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:32,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:32,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:33,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:34,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:35,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:36,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:39,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:39,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:39,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:39,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:39,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:40,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:41,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:41,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:43,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:43,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:43,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:43,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:44,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:44,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:45,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:48,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:48,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:48,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:48,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:49,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:49,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:51,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:51,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:51,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:51,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:51,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:51,948 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:52,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:53,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:54,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:55,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:55,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:55,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:55,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:56,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:56,434 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:32:56,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:57,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:59,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:59,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:59,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:59,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:59,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:00,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:01,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:03,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:03,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:03,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:03,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:03,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:05,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:05,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:06,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:07,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:07,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:07,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:07,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:07,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:09,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:10,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:11,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:11,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:11,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:11,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:11,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:11,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:13,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:14,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:14,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:14,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:14,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:15,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:15,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:17,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:18,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:18,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:18,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:18,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:19,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:21,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:21,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:21,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:22,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:22,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:22,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:22,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:22,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:23,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:25,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:25,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:25,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:25,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:26,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:26,439 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:33:26,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:27,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:30,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:30,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:30,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:30,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:30,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:31,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:33,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:34,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:34,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:34,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:34,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:35,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:36,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:38,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:41,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:42,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:42,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:42,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:42,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:43,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:46,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:46,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:46,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:46,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:47,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:47,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:48,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:51,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:51,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:51,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:53,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:54,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:55,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:55,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:55,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:55,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:55,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:56,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:56,438 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:33:57,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:58,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:58,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:58,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:58,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:59,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:59,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:01,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:04,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:06,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:07,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:07,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:07,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:07,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:07,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:09,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:10,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:11,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:15,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:15,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:15,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:15,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:15,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:16,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:17,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:19,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:19,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:19,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:19,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:19,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:20,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:21,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:25,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:26,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:26,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:26,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:26,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:26,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:26,443 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:34:26,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:27,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:30,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:30,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:30,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:30,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:30,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:31,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:33,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:34,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:34,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:34,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:34,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:36,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:36,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:37,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:37,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:37,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:37,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:37,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:39,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:40,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:40,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:40,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:40,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:41,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:41,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:41,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:43,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:46,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:46,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:46,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:47,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:47,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:49,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:49,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:49,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:49,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:50,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:51,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:51,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:52,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:53,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:53,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:53,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:53,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:53,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:55,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:56,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:56,442 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:34:58,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:58,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:58,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:58,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:59,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:01,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:02,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:02,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:02,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:02,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:03,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:03,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:05,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:06,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:06,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:06,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:06,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:06,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:07,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:07,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:09,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:09,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:09,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:09,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:09,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:10,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:11,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:11,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:13,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:13,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:13,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:13,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:14,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:14,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:15,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:17,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:17,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:17,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:17,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:17,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:19,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:20,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:20,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:20,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:20,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:21,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:21,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:23,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:25,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:25,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:25,651 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:25,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:25,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:25,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:26,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:26,447 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:35:27,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:29,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:29,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:29,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:29,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:29,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:31,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:31,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:33,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:33,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:33,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:33,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:34,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:35,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:36,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:36,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:36,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:36,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:36,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:36,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:37,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:39,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:40,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:40,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:40,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:40,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:41,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:41,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:42,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:43,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:43,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:43,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:43,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:44,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:45,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:47,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:48,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:48,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:48,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:48,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:48,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:49,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:51,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:52,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:52,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:52,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:52,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:52,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:53,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:55,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:56,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:56,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:56,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:56,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:56,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:56,446 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:35:56,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:57,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:58,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:00,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:00,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:00,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:00,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:00,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:01,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:03,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:04,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:04,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:04,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:04,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:04,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:06,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:08,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:08,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:08,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:08,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:08,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:08,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:09,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:11,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:13,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:13,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:13,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:13,058 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:13,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:14,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:15,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:19,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:21,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:22,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:22,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:22,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:22,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:23,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:25,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:25,403 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:25,403 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:25,408 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:25,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:26,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:26,452 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:36:27,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:29,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:29,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:29,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:29,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:29,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:30,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:31,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:34,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:34,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:34,436 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:34,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:34,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:34,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:35,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:39,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:39,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:39,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:39,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:39,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:40,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:41,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:43,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:43,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:44,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:45,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:50,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:52,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:52,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:52,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:52,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:52,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:53,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:56,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:56,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:56,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:56,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:56,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:56,451 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:36:56,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:56,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:57,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:01,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:02,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:02,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:02,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:02,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:03,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:03,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:05,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:06,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:10,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:11,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:11,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:11,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:11,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:11,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:12,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:13,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:15,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:15,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:15,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:15,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:17,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:17,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:19,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:19,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:19,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:19,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:19,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:20,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:21,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:23,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:23,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:23,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:23,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:23,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:23,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:25,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:25,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:26,453 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:37:27,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:27,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:27,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:27,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:27,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:29,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:29,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:30,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:30,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:30,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:30,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:30,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:31,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:34,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:35,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:36,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:36,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:36,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:36,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:36,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:37,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:39,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:39,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:39,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:39,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:39,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:40,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:40,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:41,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:43,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:43,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:43,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:43,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:43,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:45,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:45,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:49,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:49,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:49,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:49,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:49,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:51,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:51,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:52,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:52,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:52,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:52,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:53,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:53,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:55,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:56,455 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:37:56,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:56,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:56,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:56,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:56,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:56,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:57,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:59,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:59,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:59,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:59,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:00,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:01,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:01,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:03,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:03,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:03,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:03,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:03,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:05,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:05,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:07,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:08,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:08,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:08,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:08,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:08,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:09,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:10,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:11,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:11,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:11,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:11,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:11,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:13,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:13,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:15,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:15,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:15,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:15,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:15,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:17,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:18,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:19,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:19,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:20,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:21,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:23,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:23,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:24,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:25,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:25,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:26,457 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:38:27,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:27,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:27,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:27,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:27,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:29,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:31,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:31,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:31,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:31,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:31,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:33,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:35,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:35,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:35,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:35,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:35,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:35,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:36,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:37,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:40,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:40,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:40,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:40,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:40,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:40,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:40,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:41,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:44,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:44,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:44,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:44,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:44,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:45,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:49,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:49,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:49,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:49,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:49,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:50,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:51,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:51,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:55,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:55,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:55,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:56,459 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:38:57,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:57,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:58,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:58,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:58,363 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:58,368 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:58,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:59,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:02,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:02,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:02,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:02,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:02,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:03,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:03,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:05,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:07,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:07,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:07,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:07,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:07,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:08,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:09,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:10,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:11,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:11,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:13,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:14,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:14,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:14,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:14,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:14,812 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:14,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:19,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:20,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:23,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:23,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:23,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:23,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:23,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:25,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:25,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:25,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:26,461 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:39:28,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:28,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:28,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:28,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:28,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:29,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:30,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:33,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:33,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:33,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:33,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:33,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:35,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:35,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:36,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:36,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:36,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:36,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:36,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:36,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:39,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:40,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:41,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:41,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:41,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:41,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:41,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:41,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:43,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:45,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:45,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:45,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:45,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:45,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:46,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:47,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:50,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:50,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:50,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:50,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:50,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:52,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:53,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:54,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:54,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:54,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:54,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:54,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:55,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:56,466 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:39:57,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:58,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:03,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:03,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:03,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:03,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:03,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:04,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:05,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:05,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:07,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:07,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:07,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:07,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:07,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:09,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:10,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:10,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:11,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:11,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:11,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:11,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:11,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:13,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:17,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:17,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:17,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:17,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:17,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:19,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:20,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:20,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:20,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:20,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:20,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:20,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:21,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:23,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:25,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:25,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:26,465 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:40:29,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:29,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:29,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:29,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:29,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:31,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:31,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:33,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:33,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:33,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:33,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:33,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:35,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:35,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:36,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:39,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:39,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:39,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:39,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:39,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:40,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:41,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:42,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:43,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:43,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:43,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:43,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:43,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:45,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:47,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:48,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:48,819 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:48,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:48,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:48,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:50,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:51,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:52,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:53,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:53,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:53,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:53,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:53,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:55,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:55,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:56,470 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:40:57,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:57,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:57,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:57,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:57,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:57,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:00,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:02,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:03,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:03,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:03,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:03,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:04,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:05,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:06,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:07,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:07,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:08,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:08,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:10,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:10,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:10,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:10,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:10,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:10,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:11,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:12,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:13,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:14,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:14,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:14,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:14,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:15,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:18,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:18,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:19,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:19,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:19,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:19,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:20,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:20,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:22,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:24,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:25,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:26,469 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:41:27,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:27,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:27,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:27,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:29,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:30,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:31,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:31,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:31,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:31,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:32,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:34,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:34,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:34,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:34,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:34,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:35,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:35,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:35,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:38,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:40,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:40,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:43,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:43,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:43,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:43,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:44,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:45,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:46,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:46,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:47,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:47,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:47,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:47,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:48,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:50,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:51,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:51,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:51,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:51,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:52,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:54,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:55,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:55,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:55,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:55,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:56,471 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:41:57,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:58,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:00,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:00,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:00,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:00,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:00,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:01,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:02,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:03,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:05,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:07,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:07,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:07,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:07,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:08,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:08,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:10,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:10,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:13,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:15,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:15,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:15,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:15,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:16,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:18,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:18,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:19,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:19,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:19,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:19,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:20,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:20,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:22,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:22,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:22,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:22,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:22,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:23,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:24,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:24,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:25,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:26,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:26,474 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:42:28,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:29,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:29,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:29,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:29,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:30,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:30,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:32,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:35,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:36,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:36,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:36,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:36,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:36,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:38,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:39,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:39,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:39,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:39,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:40,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:40,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:40,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:42,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:43,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:43,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:43,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:43,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:44,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:45,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:46,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:46,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:47,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:47,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:47,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:47,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:48,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:50,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:50,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:50,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:50,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:50,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:50,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:51,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:51,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:54,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:54,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:54,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:54,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:54,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:55,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:55,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:56,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:56,476 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:42:57,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:59,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:59,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:59,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:59,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:00,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:02,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:02,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:03,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:03,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:03,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:04,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:04,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:05,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:06,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:08,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:08,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:08,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:08,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:08,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:10,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:10,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:11,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:11,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:11,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:11,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:12,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:13,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:14,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:15,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:15,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:15,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:15,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:16,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:18,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:18,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:19,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:19,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:19,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:19,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:20,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:20,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:22,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:24,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:24,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:24,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:24,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:24,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:24,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:25,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:26,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:26,478 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:43:28,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:28,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:28,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:28,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:29,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:29,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:30,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:30,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:34,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:34,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:34,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:35,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:35,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:36,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:38,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:38,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:38,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:38,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:39,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:40,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:40,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:40,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:42,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:42,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:42,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:42,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:43,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:44,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:46,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:46,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:46,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:46,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:46,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:47,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:48,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:50,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:50,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:50,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:50,110 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:50,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:50,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:52,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:52,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:53,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:53,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:53,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:53,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:54,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:55,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:56,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:56,480 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:43:57,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:58,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:58,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:58,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:58,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:59,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:00,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:02,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:02,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:02,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:02,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:02,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:03,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:04,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:05,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:05,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:05,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:05,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:05,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:06,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:08,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:08,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:10,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:10,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:10,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:10,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:10,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:11,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:12,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:13,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:13,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:13,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:13,350 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:14,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:14,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:16,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:16,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:16,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:16,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:16,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:17,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:20,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:20,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:20,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:20,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:20,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:20,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:20,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:21,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:22,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:25,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:25,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:25,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:25,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:25,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:25,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:26,482 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:44:28,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:29,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:29,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:29,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:29,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:30,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:30,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:32,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:35,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:35,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:35,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:35,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:35,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:36,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:36,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:38,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:39,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:39,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:39,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:39,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:40,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:40,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:42,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:42,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:47,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:47,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:47,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:47,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:47,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:48,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:50,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:50,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:52,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:52,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:52,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:52,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:52,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:53,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:54,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:55,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:56,485 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:44:58,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:58,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:58,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:58,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:58,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:59,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:00,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:03,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:03,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:03,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:03,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:03,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:04,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:05,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:06,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:08,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:08,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:08,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:08,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:08,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:09,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:10,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:10,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:11,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:11,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:11,889 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:11,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:14,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:14,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:16,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:16,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:16,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:16,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:18,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:20,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:20,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:21,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:21,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:21,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:21,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:22,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:24,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:25,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:25,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:25,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:25,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:25,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:25,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:26,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:26,490 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:45:28,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:29,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:29,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:29,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:29,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:30,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:31,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:32,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:33,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:33,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:33,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:33,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:34,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:35,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:36,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:36,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:37,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:37,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:37,665 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:37,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:38,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:40,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:40,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:41,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:41,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:41,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:41,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:42,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:44,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:45,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:45,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:46,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:47,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:48,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:49,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:49,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:49,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:49,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:50,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:50,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:52,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:52,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:55,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:56,489 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:45:56,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:56,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:56,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:56,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:57,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:57,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:02,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:04,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:04,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:04,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:04,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:04,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:05,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:06,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:07,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:07,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:07,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:07,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:08,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:08,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:10,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:10,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:11,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:11,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:11,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:11,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:12,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:13,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:14,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:16,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,803 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:16,803 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:18,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:18,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:20,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:21,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:21,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:21,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:21,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:21,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:24,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:24,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:24,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:24,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:24,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:24,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:25,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:25,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:26,493 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:46:28,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:28,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:28,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:28,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:28,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:30,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:30,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:32,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:32,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:32,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:32,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:33,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:34,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:35,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:35,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:37,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:37,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:37,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:37,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:37,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:40,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:40,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:41,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:41,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:41,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:41,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:41,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:41,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:44,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:46,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:49,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:49,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:49,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:49,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:50,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:50,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:51,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:52,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:53,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:53,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:53,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:54,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:55,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:56,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:56,492 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:46:57,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:58,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:58,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:58,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:58,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:58,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:00,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:01,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:01,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:01,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:01,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:01,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:03,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:04,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:05,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:05,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:05,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:05,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:05,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:05,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:08,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:08,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:08,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:08,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:08,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:08,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:09,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:10,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:10,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:13,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:14,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:14,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:14,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:14,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:15,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:16,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:19,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:19,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:19,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:19,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:19,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:19,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:20,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:22,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:24,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:24,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:24,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:24,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:24,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:25,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:25,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:26,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:26,498 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:47:29,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:33,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:33,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:33,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:33,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:33,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:35,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:35,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:36,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:36,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:36,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:36,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:38,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:39,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:39,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:39,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:39,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:40,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:40,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:40,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:42,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:46,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:46,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:46,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:46,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:48,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:49,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:49,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:49,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:49,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:50,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:50,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:51,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:52,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:52,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:52,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:52,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:52,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:53,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:54,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:55,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:56,497 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:47:56,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:56,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:56,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:56,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:56,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:57,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:58,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:00,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:00,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:00,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:00,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:00,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:02,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:02,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:04,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:04,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:04,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:04,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:04,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:06,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:07,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:07,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:08,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:08,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:08,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:08,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:10,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:10,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:12,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:12,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:12,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:12,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:12,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:13,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:14,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:16,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:16,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:16,597 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:16,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:17,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:18,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:18,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:20,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:22,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:22,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:22,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:22,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:23,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:23,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:24,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:25,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:26,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:26,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:26,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:26,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:26,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:26,499 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:48:28,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:29,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:34,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:34,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:34,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:34,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:34,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:35,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:35,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:36,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:37,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:37,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:37,636 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:37,641 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:38,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:39,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:40,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:40,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:41,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:41,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:41,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:41,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:41,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:42,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:45,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:46,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:46,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:46,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:46,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:46,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:48,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:50,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:50,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:53,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:53,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:53,971 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:53,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:54,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:55,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:55,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:56,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:56,501 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:48:58,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:58,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:58,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:58,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:58,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:00,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:01,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:01,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:01,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:01,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:01,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:02,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:04,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:05,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:06,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:06,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:06,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:06,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:06,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:07,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:08,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:10,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:11,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:11,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:11,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:11,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:11,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:12,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:14,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:16,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:16,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:16,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:16,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:17,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:18,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:19,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:19,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:19,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:19,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:20,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:20,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:22,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:22,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:23,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:23,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:23,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:23,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:24,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:25,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:26,503 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:49:26,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:27,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:27,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:27,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:27,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:27,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:28,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:30,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:33,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:33,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:33,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:33,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:33,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:33,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:35,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:36,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:38,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:40,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:41,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:41,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:41,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:41,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:42,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:43,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:44,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:46,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:47,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:48,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:49,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:50,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:50,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:50,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:50,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:50,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:50,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:52,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:55,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:55,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:55,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,505 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:49:56,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:58,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:00,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:02,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:02,725 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:02,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:02,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:03,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:04,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:05,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:05,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:05,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:05,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:05,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:05,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:06,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:08,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:09,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:09,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:09,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:09,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:10,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:10,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:11,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:12,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:13,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:13,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:13,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:13,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:14,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:15,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:16,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:16,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:16,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:16,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:16,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:16,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:17,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:18,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:20,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:21,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:21,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:21,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:21,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:22,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:22,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:24,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:25,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:26,507 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:50:28,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:30,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:30,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:30,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:30,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:30,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:30,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:32,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:34,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:34,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:34,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:34,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:34,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:34,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:36,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:38,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:38,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:38,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:38,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:39,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:39,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:40,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:40,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:41,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:41,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:41,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:41,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:41,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:44,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:45,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:45,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:45,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:45,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:45,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:45,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:45,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:46,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:48,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:48,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:48,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:48,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:48,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:50,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:50,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:50,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:51,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:51,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:51,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:51,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:51,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:54,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:55,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:55,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:55,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:55,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:55,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:56,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:56,510 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:50:56,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:00,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:00,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:00,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:00,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:01,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:02,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:05,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:05,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:05,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:05,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:05,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:05,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:06,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:07,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:09,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:09,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:09,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:09,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:10,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:10,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:12,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:12,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:14,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:14,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:14,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:14,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:16,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:18,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:18,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:18,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:18,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:18,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:18,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:20,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:20,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:22,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:22,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:22,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:22,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:23,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:23,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:24,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:25,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:26,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:26,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:26,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:26,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:26,512 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:51:26,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:28,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:29,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:30,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:30,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:30,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:32,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:33,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:33,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:34,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:35,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:36,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:39,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:40,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:42,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:42,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:42,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:42,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:42,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:44,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:45,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:45,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:46,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:46,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:46,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:46,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:48,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:49,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:49,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:49,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:49,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:49,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:50,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:50,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:52,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:53,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:53,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:53,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:53,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:53,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:54,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:55,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:56,517 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:51:56,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:56,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:56,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:56,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:57,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:58,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:01,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:01,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:01,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:01,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:01,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:02,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:02,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:04,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:04,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:04,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:04,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:05,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:06,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:07,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:08,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:08,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:08,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:08,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:08,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:10,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:10,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:11,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:11,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:11,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:11,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:11,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:13,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:14,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:18,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:20,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:20,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:20,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:22,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:23,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:23,302 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:23,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:23,307 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:23,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:24,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:24,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:25,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:26,516 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:52:26,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:26,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:26,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:26,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:27,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:28,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:29,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:30,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:30,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:30,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:30,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:30,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:32,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:35,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:35,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:35,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:35,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:35,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:35,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:36,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:36,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:39,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:39,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:40,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:41,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:42,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:45,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:45,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:45,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:45,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:45,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:45,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:46,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:48,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:49,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:49,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:49,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:49,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:50,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:51,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:52,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:52,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:52,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:52,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:52,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:52,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:54,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:56,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:56,521 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:52:57,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:58,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:58,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:58,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:58,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:59,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:00,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:02,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:02,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:02,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:02,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:02,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:03,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:04,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:06,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:08,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:09,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:09,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:09,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:09,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:10,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:11,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:12,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:13,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:15,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:15,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:15,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:15,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:15,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:16,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:19,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:21,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:21,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:21,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:21,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:21,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:21,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:22,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:26,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:26,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:26,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:26,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:26,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:26,520 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:53:26,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:28,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:30,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:30,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:30,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:30,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:30,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:31,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:32,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:35,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:35,164 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:35,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:35,169 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:35,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:36,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:36,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:36,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:38,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:38,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:38,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:38,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:38,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:40,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:41,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:41,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:42,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:42,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:42,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:42,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:43,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:44,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:46,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:47,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:47,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:47,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:47,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:48,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:50,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:51,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:51,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:56,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:56,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:56,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:56,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:56,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:56,525 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:53:56,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:58,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:59,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:59,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:59,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:59,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:00,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:02,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:02,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:05,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:05,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:05,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:05,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:05,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:06,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:06,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:08,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:09,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:09,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:09,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:09,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:09,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:11,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:12,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:13,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:13,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:13,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:13,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:13,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:14,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:14,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:16,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:16,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:16,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:16,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:18,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:19,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:20,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:20,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:21,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:22,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:23,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:23,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:23,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:23,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:23,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:24,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:26,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:26,524 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:54:26,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:27,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:27,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:27,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:27,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:27,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:28,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:30,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:30,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:30,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:31,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:32,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:34,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:34,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:34,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:35,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:36,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:36,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:39,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:39,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:39,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:40,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:40,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:41,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:42,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:43,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:43,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:43,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:43,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:44,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:46,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:46,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:48,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:48,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:50,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:51,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:52,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:52,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:52,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:52,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:54,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:55,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:55,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:55,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:55,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:55,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:56,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:56,526 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:54:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:58,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:01,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:01,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:01,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:01,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:01,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:02,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:03,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:05,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:06,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:06,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:08,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:08,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:08,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:08,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:08,775 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:08,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:10,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:11,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:13,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:16,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:16,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:16,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:16,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:17,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:18,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:18,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:21,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:21,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:21,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:21,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:21,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:21,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:22,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:24,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:24,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:24,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:24,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:24,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:24,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:26,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:26,528 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:55:26,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:27,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:27,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:27,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:27,711 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:27,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:29,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:30,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:31,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:31,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:31,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:31,554 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:31,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:32,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:35,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:35,649 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:35,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:35,654 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:35,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:36,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:38,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:39,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:39,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:39,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:39,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:39,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:40,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:41,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:41,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:46,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:48,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:48,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:48,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:48,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:48,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:50,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:51,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:51,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:52,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:52,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:52,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:52,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:53,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:54,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:56,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:56,531 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:55:56,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:56,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:56,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:56,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:56,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:56,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:58,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:00,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:00,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:00,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:02,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:02,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:04,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:04,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:06,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:06,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:07,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:08,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:08,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:08,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:09,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:09,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:10,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:11,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:13,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:13,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:13,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:13,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:13,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:13,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:14,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:17,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:17,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:17,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:17,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:17,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:18,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:18,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:21,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:21,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:21,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:21,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:21,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:21,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:22,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:23,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:24,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:24,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:24,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:24,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:24,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:26,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:26,533 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:56:26,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:28,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:29,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:29,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:29,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:29,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:29,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:32,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:33,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:35,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:35,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:35,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:35,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:35,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:36,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:36,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:38,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:38,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:38,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:38,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:38,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:39,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:40,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:41,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:42,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:42,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:42,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:42,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:42,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:44,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:44,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:45,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:45,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:45,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:45,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:48,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:49,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:50,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:50,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:50,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:50,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:50,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:51,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:52,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:54,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:54,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:54,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:54,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:54,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:55,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:56,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:56,535 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:56:56,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:58,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:58,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:58,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:58,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:58,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:00,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:01,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:02,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:02,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:02,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:02,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:02,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:04,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:06,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:06,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:10,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:10,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:10,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:10,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:10,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:11,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:11,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:12,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:16,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:16,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:16,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:16,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:17,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:18,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:21,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:22,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:24,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:24,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:24,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:24,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:24,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:26,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:26,537 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:57:26,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:27,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:27,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:27,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:27,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:27,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:28,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:29,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:31,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:31,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:31,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:31,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:32,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:33,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:33,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:34,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:34,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:34,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:34,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:35,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:36,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:37,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:38,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:41,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:43,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:43,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:44,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:44,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:45,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:48,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:48,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:48,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:48,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:49,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:51,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:51,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:51,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:51,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:51,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:51,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:52,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:54,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:55,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:56,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:56,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:56,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:56,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:56,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:56,539 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:57:57,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:59,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:00,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:04,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:04,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:04,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:04,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:05,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:05,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:06,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:07,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:09,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:09,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:09,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:09,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:10,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:11,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:11,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:14,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:14,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:14,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:14,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:15,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:16,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:17,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:18,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:18,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:18,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:18,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:21,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:22,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:23,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:23,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:23,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:23,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:24,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:25,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:26,544 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:58:27,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:28,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:28,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:28,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:28,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:29,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:31,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:32,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:33,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:33,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:33,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:33,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:34,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:35,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:36,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:36,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:36,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:36,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:37,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:38,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:39,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:40,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:40,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:40,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:40,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:41,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:41,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:43,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:44,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:48,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:48,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:48,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:48,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:49,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:49,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:51,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:51,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:51,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:51,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:51,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:51,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:52,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:54,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:55,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:56,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:56,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:56,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:56,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:56,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:56,543 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:58:59,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:59,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:59,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:59,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:59,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:00,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:00,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:01,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:02,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:02,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:02,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:02,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:03,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:05,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:06,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:06,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:07,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:07,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:07,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:07,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:08,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:09,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:11,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:11,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:11,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:11,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:11,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:11,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:11,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:13,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:15,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:15,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:16,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:16,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:17,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:18,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:18,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:18,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:18,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:19,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:21,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:21,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:21,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:24,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:24,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:24,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:24,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:25,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:26,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:26,548 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:59:27,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:27,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:29,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:29,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:29,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:29,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:30,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:31,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:33,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:34,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:34,192 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:34,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:34,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:35,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:36,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:37,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:37,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:37,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:37,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:37,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:38,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:38,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:39,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:40,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:40,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:40,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:40,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:41,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:41,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:43,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:43,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:45,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:45,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:45,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:45,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:46,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:47,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:48,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:51,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:51,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:51,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:51,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:51,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:52,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:53,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:54,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:55,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:55,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:55,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:55,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:56,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:56,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:56,548 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:59:57,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:00,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:01,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:01,345 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:01,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:01,350 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:02,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:03,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:06,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:06,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:06,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:06,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:06,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:06,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:09,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:10,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:10,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:10,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:10,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:10,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:11,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:12,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:13,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:15,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:15,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:15,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:15,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:17,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:17,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:20,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:20,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:20,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:20,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:20,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:21,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:23,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:23,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:25,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:25,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:25,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:25,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:26,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:26,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:26,553 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:00:27,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:28,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:30,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:30,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:30,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:30,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:31,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:33,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:33,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:36,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:38,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:39,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:39,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:39,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:39,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:40,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:41,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:41,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:42,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:42,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:42,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:42,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:43,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:43,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:45,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:47,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:47,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:48,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:49,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:49,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:51,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:51,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:51,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:51,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:51,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:52,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:53,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:54,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:56,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:56,552 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:00:57,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:57,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:57,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:57,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:58,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:59,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:00,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:01,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:01,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:01,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:01,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:01,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:03,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:04,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:04,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:04,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:04,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:05,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:06,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:06,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:07,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:07,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:07,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:09,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:11,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:11,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:11,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:11,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:11,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:11,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:12,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:13,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:15,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:15,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:15,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:15,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:16,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:16,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:17,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:20,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:20,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:20,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:20,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:21,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:21,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:21,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:23,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:26,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:26,554 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:01:27,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:29,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:29,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:29,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:29,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:29,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:31,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:33,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:33,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:33,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:33,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:34,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:35,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:36,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:38,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:38,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:38,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:38,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:38,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:41,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:41,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:41,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:41,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:41,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:41,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:42,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:43,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:43,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:45,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:45,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:45,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:45,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:46,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:47,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:49,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:51,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:53,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:53,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:53,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:53,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:54,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:54,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:55,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:56,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:56,556 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:01:57,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:57,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:57,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:57,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:58,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:59,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:00,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:01,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:01,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:01,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:01,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:01,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:03,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:05,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:05,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:05,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:05,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:06,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:06,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:06,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:07,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:09,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:09,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:09,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:09,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:11,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:11,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:11,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:13,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:13,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:13,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:13,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:14,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:15,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:16,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:17,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:17,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:17,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:17,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:18,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:19,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:21,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:23,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:23,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:23,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:23,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:23,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:25,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:26,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:26,558 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:02:26,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:26,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:26,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:26,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:27,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:27,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:29,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:32,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:35,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:35,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:35,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:35,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:36,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:36,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:37,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:38,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:39,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:39,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:39,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:39,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:40,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:41,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:41,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:43,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:43,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:43,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:43,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:44,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:45,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:46,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:46,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:46,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:46,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:47,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:49,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:50,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:50,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:50,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:50,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:50,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:51,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:53,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:53,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:53,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:53,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:53,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:54,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:54,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:56,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:56,560 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:02:58,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:58,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:58,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:58,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:59,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:59,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:01,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:02,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:02,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:02,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:02,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:03,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:05,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:05,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:06,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:06,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:06,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:06,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:06,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:09,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:11,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:11,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:11,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:11,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:11,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:11,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:12,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:13,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:16,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:17,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:17,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:17,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:17,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:17,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:19,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:22,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:25,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:25,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:25,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:25,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:26,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:26,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:26,562 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:03:27,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:27,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:29,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:29,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:29,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:29,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:31,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:32,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:32,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:32,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:32,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:32,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:33,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:35,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:36,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:36,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:36,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:36,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:36,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:36,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:37,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:38,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:39,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:39,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:39,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:39,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:40,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:41,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:41,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:43,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:44,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:44,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:44,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:44,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:44,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:45,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:47,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:47,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:47,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:47,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:47,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:49,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:49,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:51,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:51,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:51,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:54,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:54,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:54,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:54,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:54,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:55,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:56,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:56,568 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:03:57,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:58,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:58,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:58,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:58,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:59,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:00,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:02,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:02,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:02,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:02,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:02,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:05,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:06,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:06,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:07,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:07,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:07,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:07,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:07,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:09,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:10,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:10,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:10,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:10,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:11,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:11,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:11,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:13,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:15,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:15,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:15,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:15,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:16,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:16,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:20,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:20,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:20,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:20,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:20,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:21,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:22,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:23,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:25,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:25,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:25,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:25,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:25,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:26,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:26,567 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:04:27,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:27,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:28,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:28,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:28,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:28,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:29,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:31,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:32,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:32,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:32,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:33,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:33,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:36,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:36,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:36,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:36,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:36,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:37,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:38,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:39,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:40,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:40,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:40,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:40,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:40,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:41,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:41,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:44,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:44,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:44,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:44,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:44,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:47,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:49,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:49,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:49,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:49,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:49,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:49,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:51,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:51,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:54,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:54,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:54,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:54,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:54,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:55,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:56,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:56,572 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:04:57,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:58,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:58,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:58,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:58,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:59,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:01,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:01,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:01,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:01,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:01,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:03,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:05,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:05,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:05,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:05,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:05,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:06,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:06,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:07,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:10,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:10,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:10,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:10,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:10,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:11,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:11,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:11,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:14,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:14,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:14,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:14,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:14,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:15,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:17,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:20,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:20,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:20,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:20,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:21,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:23,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:23,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:23,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:23,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:23,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:24,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:25,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:26,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:26,571 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:05:26,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:26,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:26,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:26,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:27,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:27,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:29,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:30,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:30,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:30,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:30,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:30,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:33,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:36,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:38,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:38,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:38,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:38,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:38,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:39,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:41,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:41,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:42,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:42,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:42,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:42,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:43,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:43,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:45,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:46,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:46,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:46,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:46,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:46,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:49,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:49,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:51,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:54,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:55,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:55,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:55,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:55,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:55,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:56,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:56,576 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:05:57,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:59,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:04,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:04,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:04,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:04,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:04,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:05,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:05,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:06,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:07,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:07,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:07,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:07,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:07,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:09,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:10,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:11,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:11,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:11,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:11,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:11,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:11,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:13,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:15,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:15,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:15,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:15,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:16,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:17,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:21,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:21,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:23,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:23,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:23,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:23,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:24,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:25,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:26,575 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:06:26,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:26,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:26,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:26,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:26,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:27,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:29,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:31,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:31,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:31,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:31,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:31,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:32,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:33,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:35,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:35,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:35,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:35,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:35,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:36,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:37,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:38,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:39,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:39,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:39,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:39,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:39,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:41,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:41,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:43,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:44,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:44,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:44,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:44,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:44,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:45,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:47,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:47,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:48,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:49,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:49,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:51,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:52,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:52,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:52,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:52,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:52,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:53,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:54,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:55,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:55,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:55,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:55,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:56,580 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:06:56,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:57,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:59,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:59,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:59,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:59,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:59,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:00,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:01,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:06,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:06,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:06,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:06,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:06,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:06,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:07,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:09,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:09,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:09,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:09,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:10,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:10,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:11,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:11,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:16,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:16,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:16,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:16,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:16,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:17,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:20,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:20,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:20,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:20,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:20,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:21,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:21,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:23,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:24,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:24,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:24,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:24,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:25,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:26,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:26,579 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:07:26,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:27,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:31,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:33,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:33,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:33,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:33,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:33,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:35,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:36,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:37,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:37,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:37,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:37,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:37,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:37,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:39,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:40,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:40,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:40,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:40,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:40,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:41,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:41,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:42,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:45,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:45,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:45,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:45,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:46,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:47,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:47,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:49,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:49,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:49,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:49,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:51,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:51,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:53,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:53,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:53,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:53,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:53,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:53,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:55,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:56,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:56,581 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:07:57,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:57,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:57,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:57,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:57,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:58,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:59,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:01,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:01,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:01,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:01,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:01,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:03,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:04,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:04,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:04,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:04,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:04,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:05,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:06,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:07,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:08,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:08,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:08,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:08,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:08,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:09,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:10,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:11,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:12,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:12,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:12,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:12,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:12,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:15,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:16,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:16,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:16,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:19,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:20,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:21,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:21,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:21,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:21,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:21,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:21,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:23,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:26,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:26,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:26,583 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:08:27,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:27,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:27,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:27,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:27,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:29,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:31,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:31,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:31,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:31,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:31,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:31,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:33,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:34,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:34,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:34,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:34,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:35,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:36,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:36,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:37,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:38,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:38,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:38,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:38,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:38,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:39,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:41,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:42,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:43,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:43,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:43,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:43,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:43,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:45,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:46,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:46,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:46,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:46,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:46,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:47,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:50,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:50,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:50,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:50,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:50,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:51,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:51,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:53,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:53,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:53,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:53,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:53,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:54,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:55,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:56,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:56,585 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:08:57,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:57,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:57,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:57,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:57,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:58,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:59,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:01,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:01,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:01,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:01,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:01,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:03,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:04,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:05,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:05,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:05,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:05,408 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:05,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:06,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:07,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:08,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:08,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:08,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:08,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:08,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:09,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:09,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:14,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:15,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:15,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:15,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:15,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:15,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:17,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:19,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:19,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:19,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:19,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:19,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:20,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:21,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:21,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:23,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:23,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:23,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:23,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:23,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:25,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:25,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:26,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:26,587 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:09:27,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:27,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:27,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:27,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:27,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:29,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:31,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:31,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:31,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:31,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:32,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:33,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:35,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:35,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:35,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:35,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:35,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:36,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:36,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:37,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:41,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:41,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:43,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:43,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:43,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:43,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:44,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:45,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:46,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:49,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:49,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:49,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:49,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:49,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:51,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:51,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:52,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:54,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:54,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:54,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:54,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:55,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:56,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:56,589 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:09:57,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:00,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:01,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:01,302 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:01,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:01,307 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:01,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:03,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:03,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:04,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:04,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:04,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:04,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:05,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:06,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:07,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:08,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:08,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:08,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:08,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:08,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:08,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:11,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:11,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:13,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:15,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:15,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:15,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:15,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:15,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:17,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:19,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:20,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:20,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:20,301 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:20,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:20,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:21,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:21,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:25,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:26,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:26,099 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:26,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:26,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:26,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:26,595 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:10:26,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:27,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:29,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:29,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:29,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:29,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:29,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:30,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:31,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:31,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:33,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:33,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:33,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:33,644 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:33,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:35,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:36,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:36,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:37,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:37,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:37,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:37,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:38,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:39,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:41,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:41,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:41,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:41,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:41,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:41,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:42,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:43,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:46,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:46,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:46,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:46,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:46,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:47,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:48,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:49,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:49,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:49,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:51,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:51,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:53,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:54,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:54,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,780 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:54,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:56,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:56,594 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:10:57,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:59,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:00,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:00,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:00,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:00,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:01,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:05,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:05,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:05,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:05,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:05,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:05,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:06,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:08,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:08,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:09,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:10,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:11,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:11,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:11,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:11,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:11,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:11,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:13,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:15,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:15,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:15,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:15,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:15,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:15,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:17,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:20,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:21,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:21,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:21,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:21,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:21,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:21,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:23,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:25,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:25,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:25,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:25,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:25,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:26,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:26,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:26,599 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:11:27,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:29,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:29,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:29,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:29,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:30,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:30,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:31,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:31,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:34,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:34,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:34,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:34,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:34,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:35,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:36,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:37,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:40,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:40,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:40,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:40,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:40,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:42,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:43,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:44,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:44,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:44,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:44,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:44,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:45,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:49,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:49,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:49,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:49,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:49,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:49,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:51,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:51,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:54,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:54,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:54,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:54,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:54,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:54,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:55,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:56,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:56,598 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:11:58,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:58,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:58,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:58,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:59,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:02,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:02,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:02,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:02,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:03,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:06,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:06,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:10,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:10,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:10,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:10,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:11,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:11,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:11,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:13,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:14,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:14,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:14,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:14,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:14,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:17,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:17,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:18,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:18,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:18,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:18,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:18,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:19,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:21,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:22,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:22,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:22,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:22,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:22,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:23,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:23,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:26,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:26,603 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:12:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:26,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:26,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:26,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:26,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:28,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:29,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:30,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:30,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:30,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:30,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:30,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:31,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:34,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:34,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:34,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:34,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:34,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:34,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:35,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:36,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:38,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:38,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:38,567 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:38,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:38,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:39,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:39,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:41,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:44,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:46,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:46,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:46,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:46,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:46,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:49,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:49,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:51,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:51,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:51,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:51,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:51,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:51,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:53,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:54,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:54,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:54,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:54,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:54,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:55,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:55,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:56,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:56,602 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:12:59,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:59,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:59,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:59,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:59,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:01,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:01,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:04,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:04,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:04,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:04,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:04,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:05,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:06,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:07,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:07,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:07,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:07,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:07,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:07,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:09,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:10,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:10,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:10,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:10,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:10,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:11,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:11,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:12,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:14,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:14,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:14,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:14,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:14,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:15,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:17,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:20,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:20,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:20,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:21,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:21,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:23,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:24,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:24,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:24,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:24,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:24,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:26,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:26,608 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:13:27,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:28,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:28,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:28,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:28,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:28,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:29,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:29,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:33,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:33,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:33,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:33,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:33,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:34,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:35,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:36,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:38,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:38,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:38,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:38,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:38,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:40,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:40,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:41,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:41,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:41,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:41,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:41,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:42,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:44,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:45,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:47,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:47,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:47,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:47,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:48,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:50,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:50,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:50,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:50,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:50,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:50,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:51,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:51,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:54,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:55,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:56,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:56,606 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:13:59,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:59,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:59,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:59,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:00,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:01,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:02,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:03,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:03,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:03,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:03,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:04,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:06,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:06,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:07,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:07,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:07,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:07,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:07,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:08,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:10,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:11,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:12,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:12,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:12,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:12,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:12,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:13,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:14,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:16,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:16,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:16,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:16,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:17,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:18,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:18,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:20,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:20,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:20,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:20,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:21,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:21,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:23,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:24,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:26,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:26,608 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:14:27,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:27,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:27,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:27,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:28,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:29,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:30,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:31,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:31,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:31,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:31,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:32,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:34,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:35,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:35,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:35,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:35,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:35,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:36,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:38,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:40,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:40,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:40,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:40,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:40,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:41,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:41,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:42,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:43,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:43,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:43,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:43,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:44,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:45,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:46,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:50,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:51,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:52,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:52,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:52,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:52,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:53,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:54,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:56,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:56,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:56,610 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:14:58,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:58,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:58,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:00,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:02,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:02,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:02,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:02,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:02,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:03,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:04,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:06,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:07,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:07,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:07,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:07,034 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:07,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:08,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:10,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:10,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:10,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:10,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:10,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:11,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:11,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:12,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:13,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:13,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:16,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:18,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:18,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:18,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:18,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:18,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:19,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:20,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:21,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:21,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:21,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:21,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:21,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:22,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:23,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:24,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:24,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:24,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:24,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:24,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:25,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:26,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:26,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:26,612 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:15:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:28,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:28,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:28,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:28,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:29,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:30,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:34,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:34,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:34,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:34,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:35,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:36,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:36,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:38,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:38,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:38,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:38,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:39,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:39,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:40,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:41,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:43,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:43,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:43,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:43,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:43,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:43,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:45,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:47,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:47,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:47,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:47,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:47,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:49,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:50,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:51,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:51,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:51,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:51,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:51,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:52,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:53,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:55,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:55,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:55,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:56,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:56,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:56,615 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:15:57,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:59,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:59,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:59,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:59,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:00,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:00,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:01,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:01,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:04,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:04,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:04,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:04,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:05,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:05,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:06,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:07,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:07,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:07,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:07,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:07,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:08,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:09,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:11,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:12,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:14,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:14,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:14,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:14,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:15,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:15,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:15,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:17,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:17,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:17,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:17,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:17,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:18,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:19,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:21,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:21,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:21,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:21,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:21,475 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:22,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:23,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:23,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:24,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:24,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:24,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:24,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:25,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:25,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:26,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:26,617 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:16:28,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:28,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:28,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:28,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:28,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:29,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:29,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:31,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:31,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:31,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:31,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:32,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:33,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:34,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:35,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:35,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:35,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:35,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:35,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:36,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:37,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:38,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:38,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:38,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:38,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:39,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:39,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:40,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:41,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:42,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:42,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:42,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:42,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:43,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:43,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:46,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:46,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:46,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:46,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:46,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:47,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:47,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:51,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:51,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:52,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:52,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:52,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:52,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:52,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:53,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:56,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:56,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:56,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:56,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:56,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:56,619 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:16:57,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:57,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:57,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:00,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:00,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:00,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:00,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:00,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:01,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:01,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:02,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:04,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:04,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:04,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:04,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:04,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:05,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:06,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:07,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:07,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:07,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:07,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:08,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:08,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:09,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:10,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:10,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:10,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:10,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:11,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:11,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:11,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:13,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:14,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:14,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:14,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:14,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:14,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:15,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:15,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:18,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:18,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:18,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:18,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:18,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:19,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:19,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:21,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:22,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:22,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:22,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:22,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:22,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:23,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:24,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:26,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:26,621 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:17:26,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:26,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:26,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:26,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:27,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:29,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:30,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:30,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:30,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:30,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:30,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:31,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:34,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:34,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:34,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:34,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:34,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:35,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:35,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:36,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:39,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:39,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:39,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:39,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:39,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:40,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:41,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:43,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:43,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:43,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:43,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:44,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:45,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:46,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:46,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:46,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:46,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:47,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:47,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:50,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:51,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:52,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:52,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:52,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:52,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:52,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:53,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:55,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:55,408 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:55,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:55,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:55,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:56,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:56,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:56,623 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:17:57,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:59,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:59,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:59,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:59,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:00,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:01,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:01,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:03,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:03,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:03,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:03,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:04,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:05,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:06,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:06,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:06,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:06,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:06,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:06,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:07,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:07,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:10,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:10,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:10,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:10,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:11,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:11,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:12,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:16,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:16,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:16,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:16,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:16,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:17,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:18,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:20,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:20,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:20,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:20,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:20,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:21,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:23,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:23,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:23,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:23,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:23,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:24,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:25,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:26,625 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:18:27,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:27,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:27,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:27,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:27,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:29,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:29,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:30,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:30,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:30,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:30,490 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:30,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:31,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:34,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:34,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:34,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:34,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:34,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:35,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:38,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:38,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:38,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:38,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:38,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:39,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:39,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:41,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:44,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:45,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:47,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:47,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:47,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:47,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:47,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:47,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:50,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:51,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:52,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:52,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:53,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:53,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:55,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:56,631 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:18:57,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:57,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:57,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:57,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:57,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:59,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:00,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:00,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:00,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:00,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:00,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:01,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:01,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:05,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:05,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:05,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:05,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:05,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:05,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:06,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:06,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:08,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:08,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:08,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:08,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:08,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:09,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:11,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:11,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:12,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:12,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:12,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:12,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:12,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:13,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:15,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:15,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:15,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:15,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:16,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:17,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:20,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:20,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:20,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:20,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:20,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:21,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:21,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:24,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:24,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:24,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:24,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:25,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:26,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:26,629 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:19:27,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:28,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:28,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:28,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:28,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:28,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:29,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:31,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:31,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:31,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:31,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:32,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:32,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:33,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:35,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:35,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:35,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:36,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:36,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:37,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:38,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:38,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:38,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:38,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:38,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:39,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:39,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:41,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:43,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:43,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:43,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:43,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:44,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:45,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:45,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:46,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:46,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:47,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:47,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:49,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:50,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:50,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:50,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:50,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:50,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:51,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:51,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:54,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:54,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:54,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:54,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:55,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:55,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:55,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:56,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:56,634 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:19:58,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:58,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:58,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:58,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:58,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:01,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:05,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:05,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:05,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:05,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:05,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:06,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:06,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:07,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:09,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:09,465 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:09,465 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:09,470 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:09,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:11,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:11,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:12,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:13,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:13,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:13,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:13,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:18,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:18,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:18,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:18,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:18,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:19,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:19,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:21,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:23,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:26,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:26,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:26,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:26,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:26,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:26,634 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:20:26,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:27,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:29,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:30,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:30,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:30,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:30,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:30,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:31,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:35,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:35,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:35,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:35,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:35,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:35,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:35,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:36,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:40,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:40,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:40,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:40,792 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:41,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:41,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:41,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:44,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:44,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:44,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:44,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:44,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:45,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:45,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:48,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:48,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:48,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:48,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:48,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:49,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:50,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:51,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:53,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:53,241 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:53,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:53,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:53,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:56,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:56,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:56,639 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:20:58,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:58,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:58,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:58,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:59,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:01,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:01,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:01,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:01,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:01,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:01,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:03,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:06,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:06,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:10,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:10,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:10,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:10,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:10,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:11,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:11,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:12,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:13,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:13,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:13,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:13,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:13,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:13,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:17,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:17,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:17,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:17,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:17,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:17,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:18,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:20,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:20,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:20,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:20,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:20,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:21,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:21,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:23,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:26,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:26,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:26,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:26,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:26,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:26,638 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:21:26,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:27,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:28,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:31,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:31,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:31,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:31,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:34,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:34,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:34,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:34,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:34,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:35,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:35,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:36,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:39,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:39,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:39,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:39,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:39,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:39,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:40,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:41,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:44,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:44,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:44,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:44,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:44,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:45,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:49,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:49,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:49,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:49,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:49,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:49,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:51,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:51,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:52,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:52,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:52,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:52,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:52,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:53,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:56,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:56,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:56,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:56,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:56,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:56,643 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:21:56,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:57,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:01,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:01,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:01,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:01,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:02,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:05,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:05,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:05,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:05,453 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:06,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:07,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:09,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:09,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:09,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:09,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:09,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:09,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:11,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:12,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:12,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:12,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:12,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:12,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:13,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:13,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:17,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:18,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:18,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:18,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:18,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:18,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:19,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:21,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:23,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:26,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:26,642 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:22:26,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:26,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:26,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:26,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:26,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:27,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:28,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:30,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:30,470 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:30,470 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:30,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:30,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:31,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:33,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:33,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:33,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:33,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:33,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:34,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:35,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:36,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:37,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:37,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:37,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:37,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:37,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:37,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:39,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:40,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:40,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:40,930 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:40,935 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:41,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:41,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:41,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:44,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:49,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:50,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:50,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:50,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:50,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:50,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:51,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:51,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:53,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:53,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:53,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:53,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:54,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:55,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:56,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:56,645 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:22:58,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:58,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:58,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:58,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:01,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:02,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:02,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:02,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:02,406 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:02,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:03,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:06,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:07,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:07,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:07,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:07,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:07,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:08,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:09,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:11,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:12,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:12,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:12,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:12,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:12,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:12,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:13,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:17,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:17,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:17,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:17,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:17,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:17,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:19,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:20,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:20,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:20,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:20,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:20,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:21,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:21,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:22,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:24,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:24,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:24,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:24,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:24,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:25,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:26,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:26,647 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:23:27,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:27,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:27,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:27,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:27,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:27,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:28,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:30,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:30,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:30,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:30,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:31,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:31,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:33,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:34,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:34,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:34,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:34,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:34,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:35,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:36,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:38,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:39,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:39,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:39,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:39,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:39,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:39,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:41,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:42,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:42,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:42,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:42,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:42,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:43,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:44,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:45,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:46,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:46,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:46,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:46,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:46,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:47,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:49,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:50,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:50,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:50,112 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:50,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:50,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:51,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:51,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:55,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:55,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:55,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:55,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:55,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:55,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:55,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:56,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:56,649 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:23:59,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:59,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:59,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:59,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:59,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:59,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:00,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:00,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:02,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:03,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:03,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:03,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:06,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:06,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:10,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:10,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:10,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:10,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:11,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:11,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:11,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:11,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:14,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:14,687 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:14,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:14,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:14,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:15,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:15,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:17,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:21,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:22,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:23,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:23,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:23,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:23,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:23,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:23,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:26,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:26,651 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:24:26,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:26,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:26,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:26,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:27,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:27,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:27,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:30,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:30,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:30,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:30,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:30,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:31,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:33,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:35,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:35,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:35,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:35,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:35,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:35,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:36,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:38,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:38,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:38,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:38,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:38,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:39,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:39,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:41,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:41,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:41,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:41,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:41,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:41,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:43,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:44,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:45,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:45,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:45,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:45,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:45,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:45,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:49,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:49,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:49,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:49,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:49,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:49,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:50,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:51,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:52,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:52,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:52,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:52,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:52,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:53,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:55,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:56,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:56,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:56,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:56,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:56,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:56,653 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:24:56,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:57,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:00,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:01,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:05,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:05,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:05,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:05,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:05,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:05,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:06,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:07,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:08,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:08,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:08,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:08,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:08,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:09,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:11,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:12,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:15,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:15,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:15,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:15,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:15,792 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:16,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:17,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:18,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:20,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:20,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:20,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:20,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:21,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:23,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:26,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:26,658 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:25:28,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:28,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:28,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:28,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:28,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:29,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:30,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:32,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:32,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:32,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:32,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:33,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:34,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:34,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:37,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:37,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:37,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:37,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:38,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:38,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:39,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:41,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:41,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:41,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:41,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:42,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:42,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:45,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:45,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:45,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:45,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:45,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:46,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:46,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:50,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:50,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:50,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:52,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:55,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:55,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:55,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:55,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:56,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:56,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:56,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:56,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:56,657 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:25:58,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:58,999 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:59,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:59,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:59,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:00,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:02,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:03,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:03,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:03,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:03,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:04,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:06,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:06,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:06,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:06,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:06,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:07,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:07,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:08,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:11,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:12,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:12,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:12,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:12,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:13,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:13,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:14,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:17,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:17,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:17,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:17,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:18,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:18,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:18,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:20,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:20,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:20,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:20,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:20,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:21,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:22,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:23,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:24,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:24,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:24,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:24,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:25,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:25,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:26,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:26,662 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:26:28,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:28,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:28,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:28,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:29,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:29,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:30,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:30,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:32,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:32,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:33,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:34,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:35,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:38,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:38,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:38,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:38,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:39,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:40,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:40,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:40,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:45,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:46,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:47,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:48,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:50,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:50,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:50,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:50,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:50,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:50,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:51,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:52,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:54,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:54,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:54,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:54,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:55,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:55,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:55,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:56,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:56,661 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:26:58,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:58,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:58,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:58,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:59,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:00,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:01,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:01,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:01,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:01,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:01,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:02,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:02,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:05,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:05,292 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:05,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:05,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:05,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:06,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:06,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:06,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:08,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:08,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:08,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:08,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:09,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:10,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:10,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:11,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:12,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:12,523 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:12,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:12,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:13,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:14,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:16,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:16,739 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:16,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:16,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:17,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:17,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:18,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:20,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:22,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:25,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:25,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:25,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:25,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:25,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:26,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:26,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:26,666 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:27:28,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:29,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:29,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:29,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:29,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:29,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:30,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:30,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:32,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:32,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:32,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:32,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:33,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:34,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:34,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:35,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:38,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:38,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:38,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:38,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:39,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:39,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:40,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:40,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:44,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:46,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:46,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:47,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:48,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:49,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:50,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:51,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:51,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:51,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:51,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:52,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:52,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:54,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:55,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:55,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:55,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:55,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:55,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:56,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:56,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:56,665 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:27:59,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:59,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:59,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:59,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:59,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:00,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:00,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:02,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:02,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:02,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:02,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:03,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:04,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:05,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:05,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:06,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:06,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:08,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:10,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:10,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:10,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:10,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:10,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:10,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:11,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:12,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:16,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:18,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:18,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:18,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:18,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:19,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:20,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:20,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:21,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:22,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:22,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:22,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:22,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:22,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:24,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:25,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:25,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:25,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:25,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:25,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:26,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:26,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:26,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:28:26,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:29,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:29,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:29,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:29,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:30,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:30,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:32,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:35,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:35,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:35,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:35,316 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:35,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:36,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:36,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:37,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:39,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:39,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:39,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:39,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:40,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:40,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:40,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:42,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:43,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:43,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:43,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:43,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:47,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:47,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:47,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:47,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:47,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:48,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:48,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:50,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:51,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:51,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:51,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:51,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:52,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:52,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:53,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:55,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:56,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:56,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:56,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:56,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:56,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:56,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:28:58,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:59,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:00,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:01,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:01,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:01,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:01,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:01,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:02,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:04,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:04,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:04,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:04,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:05,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:05,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:05,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:06,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:08,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:08,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:08,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:08,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:09,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:10,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:10,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:11,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:12,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:12,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:12,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:12,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:13,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:14,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:15,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:15,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:15,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:15,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:15,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:16,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:16,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:18,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:20,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:20,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:20,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:20,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:20,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:21,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:22,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:22,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:24,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:25,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:25,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:26,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:26,675 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:29:27,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:28,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:28,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:28,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:28,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:29,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:30,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:31,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:31,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:31,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:31,900 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:32,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:32,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:32,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:35,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:37,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:37,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:37,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:37,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:37,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:38,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:38,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:40,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:40,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:40,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:40,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:40,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:41,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:42,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:43,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:44,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:44,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:44,308 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:44,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:45,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:46,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:48,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:50,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:50,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:50,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:50,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:50,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:51,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:52,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:53,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:55,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:55,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:55,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:55,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:55,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:55,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:56,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:56,674 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:29:58,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:58,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:58,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:58,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:58,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:59,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:00,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:00,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:03,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:03,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:03,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:03,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:03,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:04,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:04,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:05,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:08,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:08,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:08,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:08,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:08,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:09,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:10,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:10,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:12,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:12,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:12,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:12,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:12,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:14,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:15,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:15,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:15,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:15,718 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:15,718 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:15,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:16,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:16,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:20,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:20,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:20,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:20,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:20,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:20,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:21,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:22,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:24,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:24,493 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:24,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:24,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:25,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:25,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:25,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:26,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:26,679 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:30:27,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:27,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:27,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:27,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:28,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:28,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:30,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:32,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:32,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:32,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:32,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:32,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:34,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:35,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:35,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:35,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:35,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:35,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:35,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:36,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:36,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:38,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:38,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:38,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:38,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:39,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:40,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:40,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:42,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:42,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:42,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:42,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:43,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:44,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:46,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:46,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:46,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:46,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:46,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:47,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:48,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:49,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:49,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:49,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:49,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:50,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:51,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:55,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:56,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:56,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:56,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:56,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:56,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:56,678 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:30:57,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:58,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:59,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:59,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:59,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:59,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:00,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:00,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:02,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:03,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:03,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:03,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:04,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:04,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:04,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:05,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:07,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:07,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:07,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:07,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:07,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:08,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:08,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:10,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:13,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:13,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:13,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:13,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:13,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:14,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:14,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:19,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:20,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:21,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:21,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:21,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:21,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:22,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:22,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:24,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:24,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:24,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:24,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:24,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:25,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:25,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:26,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:26,680 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:31:30,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:30,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:30,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:30,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:30,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:30,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:32,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:35,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:35,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:35,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:35,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:35,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:35,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:40,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:40,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:42,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:42,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:42,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:42,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:43,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:44,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:45,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:45,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:45,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:45,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:46,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:48,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:48,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:48,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:48,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:49,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:50,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:50,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:51,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:51,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:51,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:51,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:52,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:52,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:55,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:55,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:55,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:56,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:56,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:31:56,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:59,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:59,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:59,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:59,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:00,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:00,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:02,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:07,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:07,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:07,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:07,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:07,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:10,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:11,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:11,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:11,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:11,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:12,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:12,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:12,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:15,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:15,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:15,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:15,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:16,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:16,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:18,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:19,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:19,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:19,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:19,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:20,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:20,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:20,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:23,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:23,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:23,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:23,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:23,991 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:24,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:24,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:25,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:26,684 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:32:29,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:29,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:29,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:29,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:29,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:29,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:30,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:32,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:32,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:32,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:32,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:32,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:34,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:34,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:35,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:35,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:35,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:35,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:35,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:36,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:36,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:39,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:40,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:40,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:40,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:40,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:40,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:41,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:42,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:47,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:47,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:47,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:47,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:48,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:50,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:50,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:50,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:50,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:50,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:50,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:51,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:52,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:54,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:54,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:54,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:54,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:55,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:55,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:56,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:56,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:56,686 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:00,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:00,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:00,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:01,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:01,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:02,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:05,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:05,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:05,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:05,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:05,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:05,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:06,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:07,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:09,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:09,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:09,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:09,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:10,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:10,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:10,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:12,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:13,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:13,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:13,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:13,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:13,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:14,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:18,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:20,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:21,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:21,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:21,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:21,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:22,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:23,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:25,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:26,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:26,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:26,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:26,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:26,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:26,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:26,689 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:29,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:29,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:29,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:29,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:29,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:29,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:30,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:33,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:33,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:33,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:33,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:33,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:34,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:35,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:35,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:37,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:37,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:37,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:37,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:37,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:38,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:40,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:41,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:41,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:41,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:41,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:41,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:41,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:42,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:44,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:44,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:44,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:44,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:44,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:46,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:46,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:48,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:48,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:48,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:48,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:49,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:50,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:50,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:51,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:52,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:52,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:52,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:52,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:53,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:54,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:55,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:56,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:56,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:56,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:56,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:56,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:56,700 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:57,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:00,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:00,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:00,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:01,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:02,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:03,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:03,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:03,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:03,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:03,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:04,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:04,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:05,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:08,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:09,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:09,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:09,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:09,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:09,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:10,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:10,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:14,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:16,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:16,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:18,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:19,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:19,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:19,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:19,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:19,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:20,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:20,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:20,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:23,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:23,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:23,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:23,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:24,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:24,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:25,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:26,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:26,693 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:34:27,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:27,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:27,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:27,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:27,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:28,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:30,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:32,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:32,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:32,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:32,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:32,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:34,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:35,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:35,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:36,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:36,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:36,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:36,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:36,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:38,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:40,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:40,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:41,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:41,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:41,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:41,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:41,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:42,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:45,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:45,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:45,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:45,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:45,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:46,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:48,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:50,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:50,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:50,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:50,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:50,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:50,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:51,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:52,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:55,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:56,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:56,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:56,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:56,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:56,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:56,699 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:34:56,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:58,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:59,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:59,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:59,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:59,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:01,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:02,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:03,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:03,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:03,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:03,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:03,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:04,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:05,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:07,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:07,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:07,886 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:07,890 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:08,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:10,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:10,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:12,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:12,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:12,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:12,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:12,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:13,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:14,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:15,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:15,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:16,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:18,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:19,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:19,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:19,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:19,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:19,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:20,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:20,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:24,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:24,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:24,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:24,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:24,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:25,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:25,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:26,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:26,698 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:35:28,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:28,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:28,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:28,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:28,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:30,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:30,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:32,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:32,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:32,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:32,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:34,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:35,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:36,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:36,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:36,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:36,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:36,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:37,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:38,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:40,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:40,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:40,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:40,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:40,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:41,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:41,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:42,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:45,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:46,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:47,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:47,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:47,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:47,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:47,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:48,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:51,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:51,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:51,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:51,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:51,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:51,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:51,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:52,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:56,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:56,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:56,703 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:35:59,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:59,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:59,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:59,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:00,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:00,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:01,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:04,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:04,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:04,025 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:04,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:04,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:06,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:06,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:07,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:07,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:07,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:07,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:07,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:07,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:08,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:10,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:10,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:10,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:11,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:11,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:12,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:13,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:15,066 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:16,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:18,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:20,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:20,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:21,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:22,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:23,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:23,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:23,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:23,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:23,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:24,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:24,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:26,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:26,702 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:36:28,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:28,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:28,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:28,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:28,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:29,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:31,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:31,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:31,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:31,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:31,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:32,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:34,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:36,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:39,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:39,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:39,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:39,840 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:39,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:40,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:40,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:41,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:43,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:43,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:43,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:43,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:44,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:44,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:44,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:47,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:47,209 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:47,209 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:47,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:47,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:48,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:50,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:51,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:51,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:51,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:51,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:51,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:51,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:52,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:55,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:56,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:56,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:56,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:56,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:56,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:56,707 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:36:56,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:58,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:59,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:59,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:59,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:59,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:00,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:00,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:00,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:03,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:03,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:03,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:03,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:03,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:04,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:06,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:06,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:06,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:06,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:08,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:11,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:11,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:11,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:11,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:11,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:11,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:12,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:12,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:14,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:14,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:14,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:14,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:14,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:16,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:17,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:18,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:18,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:18,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:18,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:18,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:20,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:21,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:22,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:25,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:25,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:25,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:26,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:26,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:37:26,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:27,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:29,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:29,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:29,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:30,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:30,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:33,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:33,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:33,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:33,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:33,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:33,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:34,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:36,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:38,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:40,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:40,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:40,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:40,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:40,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:40,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:41,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:43,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:43,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:43,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:43,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:43,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:43,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:44,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:48,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:51,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:52,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:52,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:52,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:52,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:52,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:54,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:55,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:55,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:55,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:55,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:55,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:56,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:56,708 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:37:56,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:59,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:00,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:00,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:00,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:00,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:02,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:04,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:04,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:04,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:04,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:04,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:05,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:06,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:06,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:07,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:07,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:07,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:07,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:07,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:08,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:10,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:11,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:12,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:12,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:12,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:12,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:12,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:14,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:16,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:16,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:16,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:16,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:16,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:18,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:19,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:19,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:19,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:19,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:19,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:20,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:21,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:21,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:23,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:23,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:23,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:23,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:24,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:24,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:26,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:26,710 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:38:26,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:28,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:28,098 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:28,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:28,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:28,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:30,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:32,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:33,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:33,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:33,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:33,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:33,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:36,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:36,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:36,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:36,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:36,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:36,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:37,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:38,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:41,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:41,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:41,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:41,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:41,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:41,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:42,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:43,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:45,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:45,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:46,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:49,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:51,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:53,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:53,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:53,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:53,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:53,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:54,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:54,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:56,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:56,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:56,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:56,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:56,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:56,713 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:38:56,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:58,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:59,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:00,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:00,187 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:00,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:00,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:02,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:04,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:04,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:04,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:04,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:04,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:05,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:06,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:06,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:07,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:07,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:07,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:07,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:07,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:08,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:10,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:11,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:15,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:16,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:16,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:16,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:16,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:16,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:16,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:20,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:20,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:20,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:20,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:20,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:20,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:21,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:21,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:23,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:23,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:23,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:23,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:23,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:24,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:26,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:26,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:26,715 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:39:27,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:27,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:27,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:27,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:27,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:28,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:30,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:30,329 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:30,329 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:30,334 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:32,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:32,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:35,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:35,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:35,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:35,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:35,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:36,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:36,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:37,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:41,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:41,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:41,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:41,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:41,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:41,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:42,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:43,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:44,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:44,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:44,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:44,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:44,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:45,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:46,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:48,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:48,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:48,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:48,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:48,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:48,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:50,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:51,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:53,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:53,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:53,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:53,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:53,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:54,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:56,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:56,717 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:39:57,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:57,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:57,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:57,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:57,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:59,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:00,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:02,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:02,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:02,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:02,577 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:04,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:05,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:06,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:08,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:08,066 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:08,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:08,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:11,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:11,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:13,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:13,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:13,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:13,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:13,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:14,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:16,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:16,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:16,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:16,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:16,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:18,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:19,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:19,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:19,898 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:19,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:19,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:20,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:21,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:21,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:23,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:23,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:23,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:23,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:23,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:24,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:26,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:26,719 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:40:26,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:26,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:26,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:26,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:26,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:27,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:28,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:31,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:31,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:31,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:31,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:31,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:32,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:33,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:35,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:36,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:36,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:36,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:36,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:36,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:38,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:38,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:40,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:40,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:40,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:40,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:40,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:40,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:40,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:43,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:43,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:43,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:43,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:43,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:43,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:44,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:45,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:47,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:47,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:47,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:47,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:48,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:49,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:50,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:51,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:51,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:51,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:51,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:52,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:53,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:54,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:54,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:54,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:54,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:54,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:55,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:55,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:56,721 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:40:57,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:58,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:58,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:58,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:58,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:59,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:59,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:00,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:02,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:02,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:02,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:02,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:03,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:05,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:05,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:05,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:10,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:10,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:10,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:10,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:10,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:10,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:11,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:13,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:14,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:14,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:14,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:14,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:15,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:15,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:17,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:18,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:18,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:18,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:18,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:19,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:19,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:20,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:22,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:22,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:22,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:22,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:23,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:23,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:25,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:26,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:26,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:26,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:26,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:26,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:26,723 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:41:26,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:27,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:30,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:30,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:30,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:30,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:31,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:31,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:32,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:35,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:35,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:35,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:35,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:35,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:36,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:37,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:38,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:39,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:39,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:39,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:39,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:40,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:40,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:41,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:43,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:45,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:45,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:46,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:47,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:48,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:48,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:48,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:48,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:48,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:49,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:50,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:51,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:52,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:52,872 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:52,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:52,877 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:53,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:53,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:55,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:55,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:56,726 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:41:57,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:57,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:57,043 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:57,048 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:57,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:59,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:59,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:00,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:00,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:00,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:00,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:01,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:03,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:04,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:04,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:04,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:04,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:04,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:05,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:05,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:07,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:08,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:08,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:08,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:08,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:09,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:09,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:10,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:11,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:12,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:12,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:12,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:12,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:13,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:15,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:15,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:17,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:17,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:17,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:17,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:18,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:19,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:20,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:20,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:21,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:21,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:21,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:21,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:22,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:23,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:25,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:25,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:25,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:25,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:25,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:26,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:26,728 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:42:26,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:27,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:29,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:29,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:29,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:29,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:30,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:30,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:31,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:32,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:34,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:34,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:34,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:34,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:35,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:35,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:37,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:37,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:39,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:39,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:39,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:39,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:40,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:40,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:41,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:43,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:43,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:43,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:43,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:43,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:45,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:48,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:50,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:51,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:51,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:51,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:51,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:52,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:53,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:53,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:55,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:55,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:55,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:55,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:55,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:55,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:56,730 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:42:57,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:58,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:59,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:59,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:59,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:59,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:00,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:01,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:04,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:05,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:05,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:05,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:05,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:05,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:06,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:07,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:09,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:10,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:10,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:10,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:10,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:10,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:11,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:11,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:15,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:15,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:15,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:15,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:15,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:16,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:19,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:19,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:19,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:19,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:20,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:20,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:20,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:22,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:22,890 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:22,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:22,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:23,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:25,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:25,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:25,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:26,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:26,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:26,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:26,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:26,735 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:43:27,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:27,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:30,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:30,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:30,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:30,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:30,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:31,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:31,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:33,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:35,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:35,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:35,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:35,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:35,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:36,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:37,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:37,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:39,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:39,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:39,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:39,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:40,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:40,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:41,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:42,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:42,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:42,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:42,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:42,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:43,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:45,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:46,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:46,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:46,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:46,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:47,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:47,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:48,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:50,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:50,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:50,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:50,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:50,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:50,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:51,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:53,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:53,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:53,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:53,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:53,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:54,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:55,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:55,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:56,734 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:43:57,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:57,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:57,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:57,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:57,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:59,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:59,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:00,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:00,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:00,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:00,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:01,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:03,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:03,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:03,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:03,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:03,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:04,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:04,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:05,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:05,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:07,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:07,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:07,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:07,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:07,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:09,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:10,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:10,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:10,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:10,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:10,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:10,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:11,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:13,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:14,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:14,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:14,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:14,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:15,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:15,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:17,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:18,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:18,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:18,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:18,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:19,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:20,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:21,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:21,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:21,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:21,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:21,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:21,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:22,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:23,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:25,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:25,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:25,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:25,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:25,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:26,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:26,739 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:44:27,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:27,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:30,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:30,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:30,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:30,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:30,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:31,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:33,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:35,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:36,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:36,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:36,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:36,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:37,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:37,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:38,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:39,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:39,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:39,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:39,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:40,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:41,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:43,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:43,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:43,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:43,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:43,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:44,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:45,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:46,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:46,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:46,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:46,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:47,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:47,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:49,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:49,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:49,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:49,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:49,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:50,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:50,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:51,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:54,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:55,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:55,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:56,738 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:44:57,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:57,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:57,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:57,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:58,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:59,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:00,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:01,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:01,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:01,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:01,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:01,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:03,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:04,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:04,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:04,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:04,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:05,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:06,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:09,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:09,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:09,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:09,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:10,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:11,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:12,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:12,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:12,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:12,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:12,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:13,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:15,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:17,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:20,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:22,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:24,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:24,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:24,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:25,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:25,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:26,744 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:45:27,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:27,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:28,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:28,258 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:28,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:28,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:28,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:29,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:31,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:31,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:31,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:31,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:32,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:32,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:33,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:35,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:35,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:35,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:35,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:35,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:36,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:37,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:38,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:39,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:39,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:39,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:39,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:40,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:40,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:41,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:43,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:43,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:43,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:43,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:43,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:44,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:45,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:46,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:46,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:46,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:46,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:47,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:49,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:49,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:50,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:50,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:50,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:50,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:50,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:50,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:51,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:54,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:54,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:54,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:54,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:55,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:55,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:55,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:55,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:56,742 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:45:58,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:58,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:58,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:58,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:59,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:59,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:01,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:02,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:02,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:02,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:02,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:05,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:05,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:06,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:10,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:11,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:11,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:11,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:11,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:11,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:12,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:13,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:15,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:15,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:15,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:15,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:15,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:17,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:17,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:20,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:20,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:20,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:20,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:20,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:20,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:21,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:23,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:23,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:23,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:23,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:23,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:24,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:25,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:25,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:26,748 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:46:28,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:30,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:30,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:30,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:30,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:31,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:33,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:33,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:34,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:34,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:34,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:34,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:35,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:35,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:35,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:39,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:40,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:41,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:43,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:44,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:44,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:44,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:44,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:44,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:45,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:45,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:50,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:52,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:52,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:52,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:52,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:55,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:56,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:46:56,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:56,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:56,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:56,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:57,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:59,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:00,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:01,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:04,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:04,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:04,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:04,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:05,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:05,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:05,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:07,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:09,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:09,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:09,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:09,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:10,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:11,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:11,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:14,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:14,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:14,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:14,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:15,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:17,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:17,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:17,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:17,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:17,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:19,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:20,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:21,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:21,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:21,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:21,417 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:21,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:22,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:25,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:26,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:26,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:26,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:26,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:26,752 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:47:27,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:32,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:32,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:32,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:32,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:32,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:33,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:35,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:38,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:38,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:38,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:38,784 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:38,789 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:39,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:40,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:41,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:43,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:44,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:44,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:44,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:44,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:44,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:45,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:45,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:47,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:47,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:47,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:47,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:48,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:48,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:49,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:50,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:51,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:51,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:51,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:51,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:51,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:53,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:54,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:54,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:54,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:54,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:54,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:55,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:55,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:55,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:56,751 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:47:59,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:00,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:00,032 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:00,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:00,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:00,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:00,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:01,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:03,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:03,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:03,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:03,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:04,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:04,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:05,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:05,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:09,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:09,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:09,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:09,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:10,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:10,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:10,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:11,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:15,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:15,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:15,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:15,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:16,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:17,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:19,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:19,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:19,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:19,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:20,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:20,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:21,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:21,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:23,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:23,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:23,301 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:23,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:23,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:25,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:25,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:26,753 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:48:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:28,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:28,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:28,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:28,879 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:29,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:31,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:32,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:33,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:33,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:33,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:33,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:33,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:35,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:35,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:36,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:36,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:36,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:36,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:37,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:38,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:39,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:40,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:40,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:40,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:40,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:40,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:41,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:41,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:43,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:43,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:43,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:43,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:43,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:44,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:45,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:45,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:48,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:49,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:49,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:49,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:49,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:50,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:50,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:51,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:52,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:52,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:52,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:52,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:53,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:53,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:54,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:55,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:56,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:56,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:56,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:56,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:56,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:56,755 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:48:57,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:59,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:59,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:59,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:59,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:00,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:00,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:00,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:01,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:04,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:04,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:04,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:04,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:04,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:05,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:05,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:06,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:10,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:11,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:13,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:13,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:13,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:13,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:13,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:15,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:17,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:17,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:17,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:17,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:17,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:17,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:19,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:20,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:20,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:20,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:20,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:20,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:21,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:21,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:22,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:25,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:26,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:26,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:26,450 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:26,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:26,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:26,757 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:49:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:27,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:30,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:30,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:30,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:30,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:30,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:31,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:33,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:33,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:33,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:33,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:33,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:34,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:35,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:35,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:38,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:40,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:42,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:42,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:42,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:42,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:42,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:43,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:44,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:45,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:45,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:45,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:47,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:49,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:49,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:49,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:49,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:49,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:50,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:50,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:51,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:54,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:54,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:54,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:54,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:54,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:55,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:56,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:56,762 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:49:58,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:58,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:58,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:58,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:59,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:59,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:01,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:02,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:02,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:02,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:02,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:02,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:03,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:05,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:05,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:05,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:05,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:06,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:06,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:07,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:07,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:10,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:10,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:10,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:11,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:11,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:11,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:12,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:15,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:15,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:15,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:15,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:17,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:18,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:19,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:19,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:19,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:19,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:20,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:21,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:21,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:23,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:24,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:24,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:24,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:24,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:24,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:25,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:26,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:26,761 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:50:28,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:28,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:28,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:28,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:28,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:29,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:29,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:34,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:34,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:34,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:34,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:34,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:35,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:35,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:36,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:38,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:38,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:38,203 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:38,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:39,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:41,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:43,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:43,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:43,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:43,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:43,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:46,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:47,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:47,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:47,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:47,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:48,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:49,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:51,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:51,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:53,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:53,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:53,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:53,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:54,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:55,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:56,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:56,766 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:50:57,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:57,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:57,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:57,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:57,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:57,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:59,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:00,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:00,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:00,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:00,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:00,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:01,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:02,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:03,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:03,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:03,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:03,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:03,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:05,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:06,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:06,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:06,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:06,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:06,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:06,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:07,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:07,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:09,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:09,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:09,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:09,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:10,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:11,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:11,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:12,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:17,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:18,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:18,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:18,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:18,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:19,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:19,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:21,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:23,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:24,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:24,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:24,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:24,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:24,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:25,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:26,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:26,766 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:51:27,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:27,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:27,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:27,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:28,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:29,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:29,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:30,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:30,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:30,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:30,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:31,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:31,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:34,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:34,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:34,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:34,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:34,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:35,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:35,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:36,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:38,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:38,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:38,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:38,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:38,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:39,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:40,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:41,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:42,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:42,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:42,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:42,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:42,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:43,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:45,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:45,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:45,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:45,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:45,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:46,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:47,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:49,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:49,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:49,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:49,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:51,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:51,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:51,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:54,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:54,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:54,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:54,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:54,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:55,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:56,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:56,771 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:51:56,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:58,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:58,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:58,407 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:58,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:58,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:59,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:02,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:02,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:02,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:02,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:02,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:02,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:03,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:06,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:06,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:06,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:06,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:06,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:06,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:07,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:08,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:10,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:10,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:10,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:10,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:10,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:11,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:11,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:13,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:15,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:15,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:17,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:19,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:19,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:19,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:19,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:19,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:19,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:19,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:21,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:23,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:23,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:23,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:23,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:23,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:24,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:25,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:26,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:26,770 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:52:26,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:26,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:26,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:26,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:27,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:27,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:29,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:30,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:30,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:30,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:31,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:34,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:34,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:34,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:34,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:34,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:35,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:35,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:36,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:37,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:37,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:37,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:37,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:38,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:39,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:40,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:41,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:41,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:41,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:41,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:41,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:41,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:43,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:45,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:45,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:45,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:45,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:46,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:47,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:49,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:49,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:49,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:49,958 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:50,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:51,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:51,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:51,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:55,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:55,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:55,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:55,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:56,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:56,775 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:52:57,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:57,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:58,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:58,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:58,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:58,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:59,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:59,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:02,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:04,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:04,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:04,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:04,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:04,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:05,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:06,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:07,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:07,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:07,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:07,813 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:07,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:08,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:09,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:11,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:12,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:12,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:12,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:12,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:12,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:13,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:14,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:15,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:15,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:15,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:15,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:15,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:17,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:19,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:19,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:19,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:19,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:19,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:19,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:20,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:21,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:22,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:22,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:22,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:22,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:23,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:23,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:25,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:26,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:26,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:53:26,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:26,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:26,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:26,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:30,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:30,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:30,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:30,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:30,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:31,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:31,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:34,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:34,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:34,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:34,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:34,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:35,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:36,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:37,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:41,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:41,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:41,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:41,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:41,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:41,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:42,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:43,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:47,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:48,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:48,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:48,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:48,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:48,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:49,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:51,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:52,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:52,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:52,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:52,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:52,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:53,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:53,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:55,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:55,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:55,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:55,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:55,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:56,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:56,779 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:53:57,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:58,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:01,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:01,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:01,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:01,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:04,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:06,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:07,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:07,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:07,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:07,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:07,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:09,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:09,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:11,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:12,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:12,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:12,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:12,091 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:12,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:13,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:15,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:15,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:15,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:15,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:15,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:15,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:16,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:17,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:20,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:20,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:20,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:20,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:20,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:20,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:21,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:21,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:23,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:23,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:23,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:23,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:26,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:26,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:26,778 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:54:28,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:28,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:28,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:29,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:32,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:32,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:32,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:32,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:32,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:32,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:33,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:36,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:37,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:37,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:37,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:37,060 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:37,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:37,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:38,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:40,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:40,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:40,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:40,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:40,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:41,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:41,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:43,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:45,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:45,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:45,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:45,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:47,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:49,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:51,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:53,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:53,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:53,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:53,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:54,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:54,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:55,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:56,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:56,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:54:58,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:58,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:58,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:58,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:58,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:59,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:00,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:05,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:06,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:07,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:07,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:07,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:07,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:07,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:07,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:10,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:10,395 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:10,395 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:10,400 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:10,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:11,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:11,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:11,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:14,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:14,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:14,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:14,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:14,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:15,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:16,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:18,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:18,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:18,816 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:18,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:18,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:19,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:21,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:22,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:22,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:22,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:22,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:22,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:22,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:23,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:26,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:26,782 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:55:27,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:27,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:27,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:27,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:27,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:28,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:29,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:30,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:30,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:30,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:30,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:30,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:31,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:33,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:35,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:35,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:35,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:35,843 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:35,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:36,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:37,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:38,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:41,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:41,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:41,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:41,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:41,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:43,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:44,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:45,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:45,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:45,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:45,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:45,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:50,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:53,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:53,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:53,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:53,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:53,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:55,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:55,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:56,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:56,784 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:55:58,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:58,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:58,182 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:58,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:58,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:59,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:00,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:01,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:01,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:01,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:01,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:01,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:02,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:04,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:06,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:06,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:06,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:06,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:06,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:06,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:07,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:08,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:09,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:09,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:09,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:09,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:10,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:11,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:11,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:12,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:16,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:18,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:18,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:18,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:18,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:19,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:20,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:21,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:22,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:24,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:24,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:24,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:24,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:25,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:26,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:26,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:26,786 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:56:27,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:27,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:27,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:27,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:28,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:32,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:32,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:32,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:32,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:33,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:33,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:34,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:36,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:36,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:36,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:36,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:36,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:37,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:38,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:38,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:41,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:41,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:41,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:41,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:41,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:42,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:43,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:44,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:45,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:48,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:50,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:50,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:50,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:50,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:51,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:51,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:52,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:54,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:54,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:54,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:54,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:54,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:55,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:56,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:56,788 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:56:57,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:57,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:57,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:57,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:58,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:59,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:00,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:00,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:01,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:01,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:01,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:01,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:02,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:04,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:04,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:04,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:04,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:04,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:05,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:06,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:06,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:08,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:08,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:08,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:08,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:09,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:10,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:11,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:11,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:11,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:11,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:11,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:11,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:12,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:14,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:15,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:15,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:16,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:16,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:17,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:18,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:18,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:18,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:18,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:19,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:20,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:21,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:22,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:22,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:22,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:23,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:24,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:26,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:26,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:26,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:26,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:26,790 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:57:27,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:28,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:28,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:32,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:32,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:32,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:32,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:33,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:34,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:34,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:37,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:37,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:37,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:37,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:38,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:38,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:40,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:41,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:41,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:41,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:41,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:41,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:42,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:44,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:45,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:45,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:45,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:45,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:45,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:46,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:46,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:49,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:49,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:49,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:49,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:50,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:50,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:51,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:51,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:52,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:52,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:52,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:52,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:53,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:54,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:56,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:56,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:56,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:56,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:56,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:56,793 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:57:56,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:57,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:58,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:00,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:00,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:00,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:00,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:01,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:02,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:02,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:04,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:04,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:04,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:04,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:04,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:06,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:06,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:07,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:07,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:07,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:07,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:07,701 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:08,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:10,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:11,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:12,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:13,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:13,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:13,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:14,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:14,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:17,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:17,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:17,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:17,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:17,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:18,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:18,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:20,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:20,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:20,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:20,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:21,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:21,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:22,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:23,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:24,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:24,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:24,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:24,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:25,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:26,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:26,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:26,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:58:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:30,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:30,278 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:31,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:32,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:34,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:35,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:35,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:35,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:35,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:36,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:36,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:36,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:38,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:38,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:38,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:38,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:39,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:39,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:40,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:41,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:42,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:42,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:42,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:42,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:43,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:44,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:45,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:46,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:46,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:46,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:46,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:46,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:48,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:51,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:51,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:51,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:51,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:51,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:51,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:52,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:52,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:54,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:54,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:54,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:54,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:55,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:56,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:56,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:56,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:56,797 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:58:58,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:58,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:58,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:58,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:59,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:00,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:02,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:02,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:02,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:02,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:02,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:03,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:04,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:05,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:05,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:05,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:05,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:06,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:06,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:07,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:08,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:09,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:09,734 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:09,734 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:09,739 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:10,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:11,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:12,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:13,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:14,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:14,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:14,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:14,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:14,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:16,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:17,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:17,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:17,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:17,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:18,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:18,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:19,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:21,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:22,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:22,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:22,705 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:22,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:23,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:24,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:24,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:26,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:26,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:59:27,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:27,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:27,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:27,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:28,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:30,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:32,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:32,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:32,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:32,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:32,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:34,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:36,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:40,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:40,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:40,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:40,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:41,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:42,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:46,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:46,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:46,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:46,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:47,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:47,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:48,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:49,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:49,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:49,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:49,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:50,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:51,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:52,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:52,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:56,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:56,801 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:59:57,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:58,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:58,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:58,091 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:58,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:58,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:00,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:02,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:02,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:02,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:02,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:03,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:03,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:04,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:05,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:05,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:05,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:05,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:06,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:06,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:06,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:08,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:09,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:09,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:09,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:09,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:10,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:11,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:12,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:13,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:13,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:13,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:13,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:13,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:14,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:16,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:18,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:18,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:18,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:18,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:18,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:19,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:20,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:21,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:22,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:22,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:22,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:22,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:23,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:24,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:24,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:26,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:26,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:26,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:26,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:26,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:26,806 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:00:27,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:28,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:30,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:33,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:33,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:33,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:33,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:34,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:34,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:36,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:36,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:36,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:36,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:36,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:36,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:37,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:38,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:39,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:39,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:39,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:39,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:40,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:41,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:41,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:42,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:46,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:48,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:48,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:48,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:48,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:48,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:50,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:50,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:52,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:53,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:53,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:53,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:53,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:54,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:54,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:55,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:56,806 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:00:56,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:56,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:56,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:56,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:57,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:57,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:58,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:00,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:00,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:00,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:01,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:02,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:03,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:04,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:04,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:04,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:05,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:05,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:06,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:08,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:08,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:08,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:08,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:09,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:09,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:10,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:10,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:13,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:13,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:13,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:13,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:14,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:14,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:15,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:18,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:18,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:18,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:18,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:18,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:20,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:20,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:21,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:21,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:21,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:21,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:22,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:24,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:24,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:24,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:24,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:25,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:25,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:26,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:26,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:26,811 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:01:28,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:28,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:28,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:28,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:28,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:30,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:30,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:31,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:31,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:31,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:31,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:31,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:32,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:34,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:35,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:36,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:36,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:36,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:36,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:37,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:37,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:38,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:39,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:39,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:39,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:39,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:40,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:40,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:42,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:43,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:43,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:43,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:43,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:44,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:44,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:45,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:47,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:47,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:47,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:47,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:47,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:48,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:48,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:50,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:53,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:53,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:53,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:53,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:53,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:53,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:54,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:55,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:56,809 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:01:57,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:57,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:57,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:57,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:58,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:58,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:58,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:01,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:01,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:01,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:01,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:02,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:02,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:03,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:05,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:05,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:05,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:05,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:05,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:05,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:06,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:09,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:10,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:10,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:10,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:10,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:10,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:11,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:12,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:14,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:14,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:14,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:14,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:14,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:14,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:16,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:17,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:17,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:17,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:17,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:18,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:18,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:19,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:20,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:21,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:21,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:21,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:21,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:21,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:22,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:24,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:24,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:24,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:24,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:24,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:25,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:26,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:26,814 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:02:28,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:28,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:28,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:28,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:28,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:30,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:30,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:30,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:31,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:31,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:31,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:31,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:32,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:32,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:34,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:34,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:34,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:34,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:35,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:35,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:35,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:36,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:38,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:38,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:38,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:38,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:39,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:40,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:40,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:41,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:45,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:46,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:47,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:47,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:47,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:47,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:47,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:48,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:50,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:50,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:50,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:50,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:51,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:51,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:52,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:55,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:55,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:55,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:55,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:55,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:55,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:56,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:56,813 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:02:57,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:58,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:58,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:58,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:58,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:58,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:00,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:01,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:01,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:01,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:01,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:02,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:02,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:03,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:05,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:05,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:05,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:05,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:05,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:05,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:06,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:09,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:10,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:10,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:10,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:10,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:10,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:11,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:12,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:13,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:13,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:13,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:13,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:14,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:14,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:16,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:18,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:18,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:18,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:18,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:18,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:20,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:20,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:20,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:21,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:21,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:21,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:21,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:22,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:22,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:24,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:24,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:24,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:24,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:25,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:25,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:25,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:26,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:26,819 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:03:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:29,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:29,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:29,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:30,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:30,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:32,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:33,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:33,649 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:33,649 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:33,654 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:34,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:34,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:35,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:36,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:37,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:37,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:37,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:37,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:37,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:38,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:40,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:42,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:43,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:43,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:43,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:43,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:44,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:44,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:45,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:47,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:47,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:47,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:47,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:47,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:48,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:48,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:50,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:51,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:51,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:51,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:51,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:51,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:52,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:53,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:54,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:54,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:54,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:54,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:55,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:55,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:56,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:56,817 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:03:58,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:00,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:00,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:00,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:00,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:00,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:02,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:04,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:04,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:04,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:04,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:04,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:04,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:05,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:06,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:08,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:08,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:08,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:08,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:08,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:10,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:10,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:11,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:12,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:12,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:13,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:14,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:15,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:15,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:15,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:15,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:15,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:15,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:16,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:18,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:19,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:19,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:19,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:19,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:19,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:20,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:21,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:21,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:24,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:24,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:24,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:24,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:25,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:26,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:26,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:26,823 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:04:26,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:28,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:28,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:28,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:28,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:30,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:31,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:31,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:31,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:31,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:31,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:32,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:32,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:36,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:37,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:40,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:40,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:40,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:40,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:41,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:41,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:42,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:42,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:43,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:43,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:43,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:43,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:43,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:44,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:45,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:47,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:47,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:47,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:47,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:47,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:48,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:48,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:50,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:50,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:50,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:50,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:50,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:51,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:52,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:53,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:53,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:53,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:53,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:53,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:54,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:54,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:56,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:56,822 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:04:58,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:58,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:58,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:58,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:58,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:59,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:00,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:02,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:02,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:02,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:02,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:02,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:04,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:05,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:05,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:05,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:05,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:05,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:05,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:06,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:06,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:10,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:11,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:14,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:14,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:14,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:14,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:14,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:15,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:16,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:19,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:19,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:19,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:19,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:19,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:20,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:21,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:21,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:23,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:23,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:23,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:23,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:23,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:26,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:26,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:26,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:26,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:26,827 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:05:27,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:30,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:31,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:31,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:31,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:31,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:32,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:34,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:34,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:34,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:34,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:35,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:36,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:36,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:37,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:37,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:37,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:37,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:37,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:37,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:38,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:41,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:41,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:41,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:41,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:41,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:41,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:42,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:43,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:45,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:45,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:45,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:45,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:46,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:49,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:51,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:51,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:51,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:51,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:51,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:51,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:52,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:54,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:54,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:54,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:54,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:54,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:54,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:56,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:56,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:56,826 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:05:58,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:58,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:58,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:58,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:58,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:59,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:02,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:02,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:02,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:02,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:02,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:04,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:05,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:06,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:06,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:06,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:06,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:06,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:06,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:08,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:09,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:09,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:09,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:09,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:10,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:10,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:10,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:11,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:13,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:13,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:13,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:13,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:13,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:14,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:16,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:21,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:21,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:22,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:22,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:22,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:22,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:22,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:24,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:25,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:25,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:25,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:25,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:25,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:26,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:26,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:26,828 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:06:26,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:29,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:29,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:29,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:29,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:30,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:30,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:32,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:36,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:36,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:36,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:36,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:36,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:36,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:38,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:38,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:39,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:39,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:39,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:39,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:41,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:43,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:43,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:43,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:43,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:43,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:44,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:44,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:49,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:49,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:49,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:49,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:49,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:49,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:50,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:51,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:53,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:53,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:53,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:53,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:54,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:54,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:54,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:56,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:56,830 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:06:57,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:57,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:57,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:57,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:57,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:58,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:00,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:01,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:01,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:01,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:01,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:02,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:02,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:05,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:06,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:06,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:06,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:06,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:06,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:06,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:08,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:09,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:09,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:09,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:09,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:09,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:10,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:11,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:11,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:12,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:12,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:12,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:14,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:16,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:16,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:16,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:16,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:16,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:18,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:19,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:19,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:19,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:19,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:20,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:20,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:21,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:21,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:24,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:24,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:24,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:24,080 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:24,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:24,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:26,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:26,832 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:07:26,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:30,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:30,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:30,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:32,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:32,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:36,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:36,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:36,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:36,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:36,480 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:36,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:38,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:38,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:39,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:39,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:39,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:39,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:41,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:43,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:43,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:43,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:43,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:43,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:44,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:44,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:48,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:48,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:48,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:48,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:48,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:49,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:50,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:51,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:51,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:51,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:51,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:51,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:52,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:52,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:54,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:56,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:56,835 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:07:59,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:00,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:00,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:00,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:02,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:04,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:04,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:04,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:04,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:04,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:05,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:06,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:06,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:09,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:09,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:09,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:09,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:09,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:10,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:11,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:15,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:15,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:15,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:15,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:16,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:16,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:20,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:21,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:21,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:21,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:21,553 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:21,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:21,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:22,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:25,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:25,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:25,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:25,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:25,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:26,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:26,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:26,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:26,836 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:08:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:30,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:30,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:30,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:30,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:31,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:32,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:35,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:35,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:35,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:35,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:35,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:36,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:36,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:37,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:38,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:38,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:38,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:38,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:39,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:40,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:41,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:42,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:43,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:43,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:43,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:43,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:43,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:44,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:47,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:47,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:47,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:47,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:47,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:48,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:48,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:51,076 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:51,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:51,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:51,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:51,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:51,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:52,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:53,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:55,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:55,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:55,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:55,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:55,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:56,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:56,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:56,841 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:08:58,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:03,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:03,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:03,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:03,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:03,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:04,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:04,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:06,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:08,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:08,390 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:08,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:08,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:08,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:09,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:10,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:11,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:13,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:13,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:13,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:13,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:13,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:14,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:14,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:18,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:18,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:18,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:18,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:18,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:20,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:20,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:21,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:25,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:25,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:25,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:25,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:25,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:26,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:26,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:26,841 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:09:26,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:29,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:29,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:29,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:29,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:29,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:30,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:31,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:33,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:33,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:33,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:33,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:34,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:36,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:36,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:39,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:39,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:39,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:39,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:39,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:40,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:41,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:42,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:42,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:42,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:42,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:42,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:43,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:44,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:46,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:46,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:46,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:46,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:46,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:47,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:48,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:49,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:49,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:49,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:49,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:49,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:50,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:51,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:53,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:55,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:55,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:55,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:55,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:55,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:56,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:56,848 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:09:56,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:58,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:59,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:59,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:59,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:59,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:59,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:00,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:03,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:03,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:03,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:03,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:03,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:04,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:04,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:06,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:06,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:06,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:06,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:06,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:06,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:08,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:09,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:09,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:09,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:09,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:09,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:09,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:10,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:11,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:12,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:12,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:12,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:12,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:13,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:14,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:14,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:19,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:21,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:21,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:21,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:21,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:21,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:21,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:22,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:25,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:26,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:26,845 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:10:29,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:29,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:29,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:29,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:29,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:30,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:35,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:36,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:37,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:37,693 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:37,693 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:37,698 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:37,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:38,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:40,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:41,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:41,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:41,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:41,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:41,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:42,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:46,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:46,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:46,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:46,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:46,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:47,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:48,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:51,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:51,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:51,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:52,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:56,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:56,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:56,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:56,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:56,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:56,850 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:10:56,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:56,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:56,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:59,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:59,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:59,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:59,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:59,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:01,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:04,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:04,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:04,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:04,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:04,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:06,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:06,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:07,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:08,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:08,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:08,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:08,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:08,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:08,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:11,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:13,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:13,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:13,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:13,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:13,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:13,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:14,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:18,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:18,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:18,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:18,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:18,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:18,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:19,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:23,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:23,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:23,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:23,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:23,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:24,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:24,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:26,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:26,849 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:11:27,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:28,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:28,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:28,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:28,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:28,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:30,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:32,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:32,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:32,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:32,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:32,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:34,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:35,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:36,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:38,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:38,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:38,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:38,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:38,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:38,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:41,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:41,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:41,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:41,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:41,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:41,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:41,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:42,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:45,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:45,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:45,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:45,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:46,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:46,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:50,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:50,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:50,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:50,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:50,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:51,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:52,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:52,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:55,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:55,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:55,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:55,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:55,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:56,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:56,855 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:11:56,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:57,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:58,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:58,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:58,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:58,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:59,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:01,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:02,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:02,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:02,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:02,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:03,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:03,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:03,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:06,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:06,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:06,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:06,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:06,590 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:07,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:08,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:10,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:10,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:10,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:10,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:11,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:11,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:11,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:14,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:14,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:14,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:14,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:14,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:15,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:15,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:19,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:19,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:19,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:19,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:20,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:20,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:21,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:21,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:25,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:26,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:26,854 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:12:27,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:27,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:27,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:27,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:29,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:30,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:30,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:30,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:30,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:31,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:33,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:34,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:34,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:34,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:34,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:35,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:35,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:36,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:36,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:38,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:38,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:38,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:38,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:39,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:41,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:41,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:46,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:46,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:46,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:46,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:46,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:47,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:49,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:51,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:52,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:52,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:52,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:52,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:53,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:53,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:56,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:56,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:56,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:56,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:56,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:56,856 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:12:56,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:57,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:57,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:00,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:00,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:00,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:00,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:01,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:02,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:03,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:06,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:07,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:09,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:09,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:10,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:11,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:11,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:13,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:13,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:13,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:13,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:13,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:15,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:18,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:18,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:18,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:18,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:18,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:19,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:21,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:21,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:22,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:22,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:22,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:22,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:23,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:24,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:25,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:25,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:25,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:25,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:26,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:26,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:26,858 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:13:27,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:29,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:29,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:29,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:29,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:29,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:30,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:31,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:32,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:32,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:32,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:32,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:33,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:35,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:35,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:38,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:38,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:38,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:38,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:39,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:39,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:41,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:41,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:41,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:42,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:43,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:45,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:45,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:45,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:45,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:46,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:46,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:47,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:48,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:48,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:48,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:48,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:49,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:51,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:51,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:52,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:52,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:52,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:52,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:52,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:53,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:55,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:56,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:56,860 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:13:57,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:59,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:59,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:59,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:59,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:00,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:01,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:03,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:03,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:03,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:03,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:03,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:03,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:05,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:06,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:06,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:06,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:06,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:06,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:07,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:07,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:08,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:11,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:13,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:16,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:16,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:16,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:16,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:17,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:17,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:19,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:21,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:21,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:21,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:21,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:21,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:22,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:23,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:24,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:25,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:25,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:25,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:25,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:25,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:26,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:26,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:14:27,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:30,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:30,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:30,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:30,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:30,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:31,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:31,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:33,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:33,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:33,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:33,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:34,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:35,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:35,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:36,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:37,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:37,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:37,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:38,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:38,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:39,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:41,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:41,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:41,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:41,655 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:41,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:41,660 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:42,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:43,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:46,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:46,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:46,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:46,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:46,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:47,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:47,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:50,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:50,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:50,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:50,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:51,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:51,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:51,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:52,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:54,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:55,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:55,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:56,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:56,864 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:14:57,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:57,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:57,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:57,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:57,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:58,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:59,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:00,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:00,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:01,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:01,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:03,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:03,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:04,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:04,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:04,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:04,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:05,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:05,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:06,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:08,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:09,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:09,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:09,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:09,928 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:10,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:11,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:11,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:13,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:15,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:15,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:15,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:15,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:16,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:17,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:19,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:19,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:19,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:19,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:19,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:20,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:21,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:21,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:24,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:24,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:24,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:24,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:24,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:25,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:25,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:26,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:26,866 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:15:29,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:29,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:29,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:29,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:29,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:30,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:31,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:32,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:32,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:32,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:32,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:33,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:33,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:35,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:36,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:37,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:37,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:37,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:37,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:37,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:39,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:41,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:41,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:42,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:42,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:42,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:42,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:43,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:43,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:45,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:45,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:45,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:45,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:45,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:46,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:46,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:47,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:48,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:48,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:48,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:48,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:49,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:49,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:51,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:51,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:54,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:54,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:54,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:54,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:55,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:56,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:56,868 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:15:56,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:58,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:58,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:58,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:58,158 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:58,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:59,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:01,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:01,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:01,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:01,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:02,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:02,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:03,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:04,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:04,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:04,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:04,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:05,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:06,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:08,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:08,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:08,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:08,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:08,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:09,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:11,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:11,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:11,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:11,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:11,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:12,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:13,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:13,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:16,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:16,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:16,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:16,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:17,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:17,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:19,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:20,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:20,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:20,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:20,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:21,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:21,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:21,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:24,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:24,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:24,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:24,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:24,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:25,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:25,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:26,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:26,870 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:16:27,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:27,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:27,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:27,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:28,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:29,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:30,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:31,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:31,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:31,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:31,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:32,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:33,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:35,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:35,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:35,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:35,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:36,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:36,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:36,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:37,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:41,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:41,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:42,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:42,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:42,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:42,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:43,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:45,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:46,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:46,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:46,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:46,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:46,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:47,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:49,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:51,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:55,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:55,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:55,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:55,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:55,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:56,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:56,872 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:16:56,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:59,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:59,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:59,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:59,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:00,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:01,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:02,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:03,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:03,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:03,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:03,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:04,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:05,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:06,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:07,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:07,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:07,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:07,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:07,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:08,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:09,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:11,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:12,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:13,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:13,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:13,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:13,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:14,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:15,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:17,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:17,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:17,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:17,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:17,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:18,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:19,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:21,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:21,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:21,182 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:21,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:21,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:21,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:23,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:23,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:25,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:25,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:25,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:25,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:26,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:26,877 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:17:27,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:28,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:29,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:29,268 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:29,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:29,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:29,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:31,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:32,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:32,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:32,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:32,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:32,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:33,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:34,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:36,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:39,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:39,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:39,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:39,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:39,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:40,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:41,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:41,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:42,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:42,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:42,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:42,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:43,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:43,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:45,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:46,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:46,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:46,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:46,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:46,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:47,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:49,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:49,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:49,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:49,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:51,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:51,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:51,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:54,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:54,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:54,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:54,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:55,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:55,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:56,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:56,877 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:17:56,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:58,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:59,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:02,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:02,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:02,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:02,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:02,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:02,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:03,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:05,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:05,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:05,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:05,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:06,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:06,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:07,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:07,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:09,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:09,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:09,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:09,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:09,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:11,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:11,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:13,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:17,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:17,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:17,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:17,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:18,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:18,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:19,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:21,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:22,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:22,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:22,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:22,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:23,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:23,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:23,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:26,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:26,882 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:18:27,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:27,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:27,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:27,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:28,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:28,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:29,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:31,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:31,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:31,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:31,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:31,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:33,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:35,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:35,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:35,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:35,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:35,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:36,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:37,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:39,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:41,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:43,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:43,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:43,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:43,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:44,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:44,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:45,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:47,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:47,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:47,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:47,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:49,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:49,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:51,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:52,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:52,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:52,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:52,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:53,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:53,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:54,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:56,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:56,881 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:18:57,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:57,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:57,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:57,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:58,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:59,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:59,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:04,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:06,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:06,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:06,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:06,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:06,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:06,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:07,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:10,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:11,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:14,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:14,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:14,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:14,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:15,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:15,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:15,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:18,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:18,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:18,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:18,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:21,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:21,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:21,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:21,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:21,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:21,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:22,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:23,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:25,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:25,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:25,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:25,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:25,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:26,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:26,886 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:19:26,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:27,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:28,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:28,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:28,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:28,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:29,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:29,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:32,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:33,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:33,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:33,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:33,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:34,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:35,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:36,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:37,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:37,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:37,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:37,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:37,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:38,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:39,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:41,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:42,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:45,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:45,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:45,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:45,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:46,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:48,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:48,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:48,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:48,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:48,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:49,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:49,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:51,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:52,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:52,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:52,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:52,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:53,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:53,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:54,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:56,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:56,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:56,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:56,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:56,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:56,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:56,885 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:19:57,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:59,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:59,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:59,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:59,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:59,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:00,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:01,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:04,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:06,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:06,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:06,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:06,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:06,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:06,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:07,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:09,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:09,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:09,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:09,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:09,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:10,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:11,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:11,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:15,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:17,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:17,268 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:17,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:17,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:17,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:19,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:21,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:21,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:21,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:21,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:21,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:21,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:22,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:23,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:25,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:25,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:25,311 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:25,316 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:25,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:26,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:26,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:26,891 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:20:27,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:28,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:28,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:28,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:28,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:31,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:31,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:31,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:31,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:31,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:32,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:33,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:36,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:36,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:37,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:37,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:37,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:37,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:37,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:39,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:41,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:42,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:45,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:45,732 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:45,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:46,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:47,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:47,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:51,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:52,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:53,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:53,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:53,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:53,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:54,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:55,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:56,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:56,889 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:20:57,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:57,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:57,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:57,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:57,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:58,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:59,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:01,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:01,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:01,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:01,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:02,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:03,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:03,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:06,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:06,046 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:06,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:06,051 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:06,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:06,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:07,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:09,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:09,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:09,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:09,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:09,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:10,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:11,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:11,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:13,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:13,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:13,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:13,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:13,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:14,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:15,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:18,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:18,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:18,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:18,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:19,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:19,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:19,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:21,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:22,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:22,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:22,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:22,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:22,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:23,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:25,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:25,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:25,914 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:25,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:25,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:26,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:26,892 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:21:27,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:29,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:29,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:29,567 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:29,571 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:29,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:30,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:31,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:33,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:33,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:33,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:33,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:34,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:35,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:35,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:36,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:37,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:37,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:37,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:37,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:38,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:39,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:40,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:41,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:43,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:43,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:43,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:43,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:44,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:45,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:46,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:47,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:47,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:47,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:47,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:47,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:49,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:51,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:52,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:55,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:55,693 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:55,693 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:55,698 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:56,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:56,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:56,894 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:21:57,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:57,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:59,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:59,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:59,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:59,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:59,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:01,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:02,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:02,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:02,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:02,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:02,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:03,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:03,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:05,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:05,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:05,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:05,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:05,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:06,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:07,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:08,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:09,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:09,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:09,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:09,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:09,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:09,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:11,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:14,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:14,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:14,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:14,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:15,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:18,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:18,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:18,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:18,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:18,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:19,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:19,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:21,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:23,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:23,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:23,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:23,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:23,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:24,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:25,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:26,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:26,896 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:22:27,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:27,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:27,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:27,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:27,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:27,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:30,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:31,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:31,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:31,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:31,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:31,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:33,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:34,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:34,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:34,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:34,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:35,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:35,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:36,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:39,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:39,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:39,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:39,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:39,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:41,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:41,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:41,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:44,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:44,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:44,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:44,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:45,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:46,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:51,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:51,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:53,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:53,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:53,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:53,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:53,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:55,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:56,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:56,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:56,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:56,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:56,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:56,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:56,898 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:22:56,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:57,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:01,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:01,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:01,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:01,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:01,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:02,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:03,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:05,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:05,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:05,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:05,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:05,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:05,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:06,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:08,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:08,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:08,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:08,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:08,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:09,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:09,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:11,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:11,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:11,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:11,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:11,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:12,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:13,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:13,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:17,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:17,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:17,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:17,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:17,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:17,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:19,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:21,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:21,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:21,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:21,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:21,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:21,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:23,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:24,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:25,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:25,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:25,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:25,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:25,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:25,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:26,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:26,900 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:23:29,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:29,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:29,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:29,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:29,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:29,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:30,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:33,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:33,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:33,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:33,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:33,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:33,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:35,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:36,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:39,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:39,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:39,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:39,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:41,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:41,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:42,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:42,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:42,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:42,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:42,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:43,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:46,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:47,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:47,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:47,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:51,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:52,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:55,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:55,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:55,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:55,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:55,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:55,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:56,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:56,906 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:23:57,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:02,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:02,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:02,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:02,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:03,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:03,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:03,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:06,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:06,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:06,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:06,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:06,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:07,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:09,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:11,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:11,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:11,595 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:11,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:11,600 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:11,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:13,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:14,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:17,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:17,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:17,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:17,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:20,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:20,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:20,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:20,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:20,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:20,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:21,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:21,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:24,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:24,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:24,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:24,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:24,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:25,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:25,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:26,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:26,905 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:24:30,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:30,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:30,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:30,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:30,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:31,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:31,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:33,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:33,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:34,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:34,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:35,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:36,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:37,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:37,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:37,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:37,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:37,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:37,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:39,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:41,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:42,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:42,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:42,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:42,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:42,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:42,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:43,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:45,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:45,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:45,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:45,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:45,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:45,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:47,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:47,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:50,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:50,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:50,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:50,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:50,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:51,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:52,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:56,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:24:57,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:57,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:57,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:57,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:57,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:58,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:59,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:01,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:01,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:01,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:03,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:04,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:05,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:06,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:06,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:06,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:06,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:06,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:07,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:09,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:09,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:09,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:09,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:09,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:09,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:10,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:11,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:13,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:13,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:13,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:13,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:13,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:14,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:15,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:16,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:16,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:16,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:16,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:17,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:19,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:20,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:20,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:21,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:21,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:21,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:21,706 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:21,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:23,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:25,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:25,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:25,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:25,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:25,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:25,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:25,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:26,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:25:27,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:29,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:29,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:29,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:29,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:29,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:31,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:31,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:32,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:32,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:32,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:32,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:32,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:33,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:35,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:36,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:39,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:39,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:39,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:39,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:39,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:40,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:41,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:42,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:43,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:43,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:43,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:43,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:43,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:45,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:48,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:49,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:49,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:49,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:49,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:49,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:50,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:51,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:53,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:53,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:53,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:53,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:53,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:54,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:55,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:55,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:56,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:25:58,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:58,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:58,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:58,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:58,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:59,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:03,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:05,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:06,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:06,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:06,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:06,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:07,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:07,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:08,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:10,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:10,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:10,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:10,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:10,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:11,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:13,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:13,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:14,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:14,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:14,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:14,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:14,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:15,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:18,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:18,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:18,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:18,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:18,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:19,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:19,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:20,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:22,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:22,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:22,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:22,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:23,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:23,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:24,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:25,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:26,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:26,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:26,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:26,406 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:26,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:26,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:26:27,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:29,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:29,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:29,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:29,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:29,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:31,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:33,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:33,221 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:33,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:33,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:33,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:35,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:35,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:36,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:38,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:38,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:38,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:38,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:38,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:39,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:40,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:41,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:41,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:41,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:41,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:41,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:41,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:43,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:45,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:45,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:45,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:45,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:45,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:47,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:47,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:50,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:51,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:51,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:51,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:51,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:51,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:53,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:53,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:55,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:55,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:55,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:55,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:55,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:55,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:56,920 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:26:57,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:58,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:59,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:59,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:59,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:59,284 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:59,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:01,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:03,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:03,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:03,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:03,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:03,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:04,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:05,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:05,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:08,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:08,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:08,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:08,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:08,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:09,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:10,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:10,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:13,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:13,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:13,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:13,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:13,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:15,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:16,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:18,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:18,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:18,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:18,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:18,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:19,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:20,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:21,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:25,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:26,919 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:27:27,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:27,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:27,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:27,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:27,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:27,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:29,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:31,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:31,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:31,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:31,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:31,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:33,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:33,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:34,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:34,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:34,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:34,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:34,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:35,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:35,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:38,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:40,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:40,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:40,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:40,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:40,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:40,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:41,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:43,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:44,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:44,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:44,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:44,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:44,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:45,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:48,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:48,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:48,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:48,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:48,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:49,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:49,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:50,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:51,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:51,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:51,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:51,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:53,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:54,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:54,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:54,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:54,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:54,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:54,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:55,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:55,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:56,924 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:27:58,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:58,694 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:58,694 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:58,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:58,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:59,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:00,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:02,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:02,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:02,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:02,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:02,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:03,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:05,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:06,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:07,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:07,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:10,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:11,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:11,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:11,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:11,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:11,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:11,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:13,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:15,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:15,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:15,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:15,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:16,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:17,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:18,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:19,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:19,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:19,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:19,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:20,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:20,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:23,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:23,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:23,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:23,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:23,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:24,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:25,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:26,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:26,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:26,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:26,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:26,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:26,923 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:28:27,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:28,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:28,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:31,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:31,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:31,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:31,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:32,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:34,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:34,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:34,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:34,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:34,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:34,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:35,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:35,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:36,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:39,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:40,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:42,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:42,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:42,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:42,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:43,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:44,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:44,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:46,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:46,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:46,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:46,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:47,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:48,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:49,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:49,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:49,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:49,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:50,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:50,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:52,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:54,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:54,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:55,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:55,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:56,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:56,925 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:28:56,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:58,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:58,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:59,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:00,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:02,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:02,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:02,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:02,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:02,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:03,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:04,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:05,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:06,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:06,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:06,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:06,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:07,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:07,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:08,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:10,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:10,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:10,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:10,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:10,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:11,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:12,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:12,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:16,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:16,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:16,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:16,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:17,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:18,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:18,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:19,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:19,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:19,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:19,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:20,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:20,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:22,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:23,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:24,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:24,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:24,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:24,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:25,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:25,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:26,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:26,927 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:29:28,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:28,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:28,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:28,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:29,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:30,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:31,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:31,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:31,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:31,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:32,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:34,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:34,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:35,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:35,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:35,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:35,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:36,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:36,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:38,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:39,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:40,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:40,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:40,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:41,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:41,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:42,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:44,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:46,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:46,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:46,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:46,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:47,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:48,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:50,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:50,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:50,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:50,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:50,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:51,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:51,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:52,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:55,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:56,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:56,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:56,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:56,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:56,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:56,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:56,930 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:29:58,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:00,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:00,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:00,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:00,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:01,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:02,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:05,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:05,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:05,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:05,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:05,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:06,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:06,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:08,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:09,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:09,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:09,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:09,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:10,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:11,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:11,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:12,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:12,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:12,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:12,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:12,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:13,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:14,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:16,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:16,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:17,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:18,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:21,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:22,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:24,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:24,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:24,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:24,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:25,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:26,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:26,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:26,932 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:30:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:27,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:27,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:27,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:27,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:28,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:30,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:31,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:31,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:31,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:31,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:32,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:33,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:34,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:37,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:37,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:37,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:37,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:37,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:38,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:39,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:41,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:41,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:42,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:44,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:44,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:47,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:47,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:47,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:47,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:48,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:48,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:50,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:51,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:51,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:51,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:51,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:54,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:55,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:56,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:56,934 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:30:58,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:58,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:58,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:58,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:59,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:00,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:01,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:02,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:02,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:02,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:02,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:03,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:04,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:06,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:06,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:06,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:06,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:06,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:07,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:07,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:08,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:10,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:10,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:10,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:10,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:10,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:11,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:12,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:13,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:18,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:18,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:18,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:18,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:18,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:19,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:20,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:21,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:23,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:23,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:23,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:23,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:23,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:24,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:26,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:26,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:26,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:26,794 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:26,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:26,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:26,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:31:27,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:28,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:28,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:31,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:31,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:31,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:31,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:32,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:34,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:34,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:35,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:35,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:35,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:35,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:36,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:36,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:38,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:38,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:38,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:38,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:38,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:39,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:39,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:40,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:41,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:42,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:42,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:42,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:42,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:43,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:44,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:45,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:46,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:46,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:46,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:46,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:47,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:48,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:49,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:49,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:49,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:49,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:50,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:50,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:51,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:52,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:53,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:53,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:53,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:53,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:53,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:54,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:56,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:56,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:56,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:56,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:56,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:56,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:56,939 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:31:57,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:58,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:00,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:00,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:00,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:00,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:00,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:02,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:02,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:06,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:07,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:09,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:09,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:09,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:09,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:10,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:11,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:12,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:12,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:12,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:12,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:13,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:14,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:16,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:16,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:16,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:16,400 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:17,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:17,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:18,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:20,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:20,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:20,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:20,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:21,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:22,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:23,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:23,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:23,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:23,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:23,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:24,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:26,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:26,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:26,941 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:32:27,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:27,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:27,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:27,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:28,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:28,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:30,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:30,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:30,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:30,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:31,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:32,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:33,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:35,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:35,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:35,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:36,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:36,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:36,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:39,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:41,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:43,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:43,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:43,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:43,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:44,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:44,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:46,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:49,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:49,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:49,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:49,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:50,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:50,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:51,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:52,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:53,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:53,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:53,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:53,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:54,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:54,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:56,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:56,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:56,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:56,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:56,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:56,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:56,943 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:32:57,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:58,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:01,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:02,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:02,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:02,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:02,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:02,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:04,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:05,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:05,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:05,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:05,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:06,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:06,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:07,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:08,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:08,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:08,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:08,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:08,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:09,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:10,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:11,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:12,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:17,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:17,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:17,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:17,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:17,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:18,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:18,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:20,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:20,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:20,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:20,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:20,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:21,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:22,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:24,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:25,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:25,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:25,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:25,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:26,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:26,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:26,948 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:33:29,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:33,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:33,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:33,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:33,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:34,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:35,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:36,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:36,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:38,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:38,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:38,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:38,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:39,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:40,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:40,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:41,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:42,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:42,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:42,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:42,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:43,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:44,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:46,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:46,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:46,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:46,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:47,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:48,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:51,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:51,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:53,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:53,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:53,816 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:53,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:54,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:56,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:56,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:56,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:56,947 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:34:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:01,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:02,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:02,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:02,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:02,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:02,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:04,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:06,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:07,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:07,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:07,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:07,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:07,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:08,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:08,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:10,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:10,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:10,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:10,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:11,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:11,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:12,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:12,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:16,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:16,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:16,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:16,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:17,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:18,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:18,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:19,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:19,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:19,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:19,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:20,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:21,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:22,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:23,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:23,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:23,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:23,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:23,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:24,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:24,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:26,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:26,953 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:34:28,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:28,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:28,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:28,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:28,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:29,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:30,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:33,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:33,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:33,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:33,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:34,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:36,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:36,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:39,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:40,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:40,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:40,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:40,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:40,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:41,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:42,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:44,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:44,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:44,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:44,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:44,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:50,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:51,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:52,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:52,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:52,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:52,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:53,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:54,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:55,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:56,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:56,951 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:01,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:01,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:01,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:01,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:02,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:04,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:06,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:06,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:06,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:06,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:06,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:06,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:07,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:10,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:10,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:10,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:10,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:10,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:11,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:12,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:12,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:14,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:14,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:14,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:14,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:15,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:16,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:17,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:19,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:19,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:19,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:19,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:20,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:21,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:23,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:24,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:24,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:24,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:24,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:24,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:26,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:26,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:26,954 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:27,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:27,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:27,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:27,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:28,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:28,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:28,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:33,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:35,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:35,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:35,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:35,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:35,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:36,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:36,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:39,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:39,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:39,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:39,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:40,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:41,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:42,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:43,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:43,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:43,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:43,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:44,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:44,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:46,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:46,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:46,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:46,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:46,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:48,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:49,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:50,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:50,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:50,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:50,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:50,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:51,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:52,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:55,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:55,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:55,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:55,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:55,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:56,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:56,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:56,956 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:58,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:01,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:01,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:01,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:01,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:02,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:04,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:05,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:05,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:05,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:05,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:05,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:06,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:06,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:06,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:10,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:10,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:10,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:10,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:11,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:11,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:11,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:12,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:15,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:15,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:15,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:17,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:21,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:21,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:21,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:21,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:21,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:22,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:22,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:24,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:25,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:25,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:25,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:25,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:26,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:26,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:26,958 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:36:27,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:28,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:29,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:29,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:29,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:30,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:32,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:33,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:36,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:38,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:38,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:38,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:38,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:38,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:39,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:40,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:41,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:43,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:43,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:43,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:43,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:43,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:44,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:45,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:45,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:47,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:47,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:47,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:47,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:48,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:50,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:50,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:51,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:52,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:52,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:52,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:52,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:52,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:54,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:56,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:56,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:56,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:56,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:56,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:56,960 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:36:57,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:58,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:00,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:00,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:00,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:00,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:01,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:01,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:02,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:04,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:04,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:04,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:04,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:04,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:06,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:06,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:07,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:10,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:10,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:10,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:10,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:10,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:11,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:12,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:12,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:14,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:14,792 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:14,792 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:14,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:15,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:16,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:17,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:19,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:19,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:19,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:19,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:19,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:20,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:21,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:22,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:22,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:22,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:22,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:22,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:23,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:24,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:26,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:26,963 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:37:27,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:27,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:27,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:27,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:27,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:28,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:28,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:31,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:31,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:31,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:31,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:31,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:32,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:34,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:36,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:36,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:36,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:36,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:36,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:36,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:38,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:40,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:40,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:40,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:40,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:40,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:41,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:41,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:42,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:44,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:44,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:44,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:44,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:45,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:45,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:46,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:48,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:48,542 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:48,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:48,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:48,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:50,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:51,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:51,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:54,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:54,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:54,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:54,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:54,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:56,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:56,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:56,965 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:37:56,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:57,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:57,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:57,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:57,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:58,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:58,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:00,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:01,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:01,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:01,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:01,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:01,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:02,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:02,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:04,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:04,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:04,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:04,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:05,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:06,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:06,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:07,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:08,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:08,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:08,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:08,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:09,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:10,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:11,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:12,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:12,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:12,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:12,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:12,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:13,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:14,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:16,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:16,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:16,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:16,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:16,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:18,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:18,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:20,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:20,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:20,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:20,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:20,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:21,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:22,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:24,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:24,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:24,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:24,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:24,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:24,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:26,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:26,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:26,967 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:38:29,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:29,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:29,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:29,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:29,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:29,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:30,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:34,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:34,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:34,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:34,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:34,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:35,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:36,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:36,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:39,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:39,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:39,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:39,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:39,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:40,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:40,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:41,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:42,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:42,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:42,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:42,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:42,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:44,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:46,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:46,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:46,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:46,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:46,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:48,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:49,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:49,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:49,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:49,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:49,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:50,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:50,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:51,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:55,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:55,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:55,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:55,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:55,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:56,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:56,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:56,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:56,970 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:38:59,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:59,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:59,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:59,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:01,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:02,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:02,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:02,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:02,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:03,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:04,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:06,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:06,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:06,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:06,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:06,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:06,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:07,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:08,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:11,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:11,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:11,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:11,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:11,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:12,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:12,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:12,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:15,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:16,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:16,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:16,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:16,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:18,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:18,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:20,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:20,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:20,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:20,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:21,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:21,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:22,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:23,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:25,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:25,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:25,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:25,943 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:26,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:26,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:26,972 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:39:28,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:28,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:30,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:30,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:30,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:30,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:30,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:30,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:32,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:34,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:34,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:34,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:34,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:34,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:34,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:35,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:36,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:37,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:37,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:37,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:37,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:38,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:38,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:39,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:40,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:40,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:40,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:40,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:40,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:40,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:42,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:44,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:44,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:44,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:44,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:44,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:45,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:46,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:47,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:47,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:47,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:47,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:47,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:48,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:50,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:50,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:51,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:51,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:51,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:51,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:51,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:52,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:55,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:55,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:55,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:55,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:55,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:55,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:56,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:56,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:56,974 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:39:59,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:59,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:59,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:59,244 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:59,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:00,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:02,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:03,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:03,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:03,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:03,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:03,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:04,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:05,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:07,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:08,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:08,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:08,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:08,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:09,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:10,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:10,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:12,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:12,403 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:12,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:12,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:12,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:13,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:14,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:15,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:15,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:15,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:15,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:15,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:16,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:18,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:19,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:19,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:19,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:19,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:19,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:20,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:20,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:22,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:22,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:22,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:22,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:23,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:23,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:24,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:25,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:25,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:25,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:25,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:25,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:26,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:26,976 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:40:28,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:28,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:30,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:30,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:30,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:30,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:30,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:32,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:34,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:35,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:39,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:39,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:39,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:39,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:39,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:40,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:40,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:40,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:42,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:42,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:42,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:42,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:43,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:44,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:46,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:46,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:46,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:46,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:46,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:48,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:50,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:51,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:54,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:54,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:54,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:54,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:54,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:55,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:56,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:56,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:56,981 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:40:59,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:59,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:59,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:59,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:59,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:00,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:02,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:02,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:02,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:02,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:02,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:02,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:04,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:05,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:06,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:06,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:06,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:06,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:06,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:08,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:08,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:09,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:09,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:09,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:09,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:09,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:10,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:10,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:12,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:12,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:12,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:12,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:13,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:13,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:14,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:18,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:20,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:20,651 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:20,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:20,656 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:20,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:20,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:22,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:24,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:24,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:24,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:25,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:25,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:26,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:26,980 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:41:29,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:29,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:29,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:29,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:29,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:30,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:30,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:33,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:33,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:33,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:33,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:33,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:34,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:35,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:36,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:37,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:37,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:37,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:37,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:37,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:38,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:40,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:41,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:41,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:41,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:41,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:41,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:41,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:42,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:44,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:44,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:44,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:44,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:46,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:46,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:48,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:48,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:48,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:48,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:48,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:50,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:50,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:52,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:52,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:52,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:52,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:52,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:54,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:55,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:56,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:56,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:56,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:56,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:56,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:56,986 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:41:57,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:58,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:59,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:59,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:59,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:59,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:59,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:02,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:02,916 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:02,916 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:02,921 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:03,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:03,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:04,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:05,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:06,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:06,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:06,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:06,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:06,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:08,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:09,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:10,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:11,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:11,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:11,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:11,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:12,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:12,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:14,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:15,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:16,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:16,436 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:16,436 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:16,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:16,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:18,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:20,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:20,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:20,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:20,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:20,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:22,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:23,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:23,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:23,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:23,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:23,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:24,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:25,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:25,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:26,985 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:42:27,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:27,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:27,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:27,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:27,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:28,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:30,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:30,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:30,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:30,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:30,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:31,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:32,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:35,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:35,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:39,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:39,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:39,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:39,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:40,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:40,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:41,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:43,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:43,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:43,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:43,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:43,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:44,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:46,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:47,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:47,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:47,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:47,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:47,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:48,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:50,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:52,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:53,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:53,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:53,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:53,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:53,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:54,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:55,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:56,990 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:42:57,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:57,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:57,348 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:57,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:57,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:58,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:03,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:05,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:05,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:05,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:05,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:05,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:05,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:06,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:08,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:09,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:09,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:10,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:10,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:14,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:14,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:14,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:14,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:14,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:15,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:16,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:19,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:20,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:20,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:20,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:20,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:20,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:20,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:22,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:23,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:23,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:23,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:23,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:23,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:24,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:24,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:25,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:26,989 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:43:29,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:29,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:29,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:29,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:29,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:30,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:33,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:33,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:33,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:33,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:33,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:34,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:35,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:35,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:37,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:37,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:37,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:37,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:37,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:38,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:40,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:41,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:43,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:43,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:43,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:43,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:44,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:46,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:46,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:46,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:46,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:46,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:47,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:48,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:49,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:49,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:49,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:50,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:50,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:52,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:53,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:53,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:53,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:53,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:53,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:54,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:55,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:56,995 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:43:57,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:58,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:58,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:58,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:58,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:58,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:01,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:02,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:02,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:02,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:02,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:03,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:03,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:05,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:05,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:07,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:07,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:07,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:07,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:08,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:08,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:09,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:10,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:10,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:10,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:10,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:11,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:13,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:13,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:14,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:14,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:14,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:14,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:15,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:17,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:18,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:18,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:18,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:18,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:18,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:19,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:21,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:21,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:23,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:23,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:23,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:23,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:24,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:24,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:26,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:26,994 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:44:29,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:32,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:32,074 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:32,074 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:32,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:33,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:33,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:35,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:35,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:35,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:35,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:35,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:36,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:36,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:37,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:40,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:40,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:40,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:40,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:40,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:41,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:41,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:43,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:44,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:44,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:44,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:44,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:45,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:45,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:46,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:50,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:50,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:50,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:50,513 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:51,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:51,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:51,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:53,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:53,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:53,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:53,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:53,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:54,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:55,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:56,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:56,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:56,999 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:44:57,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:57,279 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:57,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:57,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:58,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:59,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:00,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:01,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:01,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:01,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:01,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:01,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:02,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:03,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:06,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:06,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:06,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:06,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:06,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:06,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:07,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:09,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:10,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:10,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:11,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:11,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:12,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:13,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:13,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:13,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:13,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:14,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:15,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:17,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:17,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:17,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:17,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:17,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:18,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:19,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:20,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:20,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:20,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:20,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:21,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:21,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:23,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:23,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:24,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:24,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:24,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:24,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:24,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:25,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:26,998 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:45:28,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:28,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:28,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:28,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:28,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:29,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:31,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:31,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:31,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:31,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:31,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:32,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:33,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:33,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:35,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:35,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:35,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:35,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:36,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:36,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:37,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:38,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:39,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:39,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:39,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:39,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:39,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:41,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:41,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:42,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:42,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:42,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:42,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:43,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:44,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:45,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:46,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:46,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:46,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:46,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:50,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:50,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:50,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:50,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:50,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:51,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:51,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:53,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:54,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:54,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:54,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:54,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:55,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:55,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:55,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:56,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:57,003 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:00,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:01,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:01,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:01,828 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:01,828 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:01,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:02,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:03,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:04,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:04,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:04,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:05,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:05,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:06,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:07,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:07,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:10,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:10,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:10,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:10,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:10,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:11,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:11,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:12,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:14,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:14,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:14,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:14,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:14,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:15,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:17,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:18,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:18,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:18,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:18,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:19,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:19,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:21,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:21,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:21,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:21,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:21,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:22,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:22,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:23,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:25,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:25,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:25,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:25,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:26,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:26,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:27,002 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:27,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:28,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:29,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:29,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:29,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:29,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:30,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:31,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:32,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:32,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:32,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:32,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:33,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:33,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:35,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:36,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:37,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:37,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:38,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:39,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:39,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:41,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:41,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:41,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:41,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:41,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:41,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:43,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:45,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:45,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:45,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:45,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:45,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:47,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:49,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:49,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:49,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:49,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:49,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:50,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:51,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:52,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:52,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:52,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:52,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:53,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:55,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:55,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:56,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:56,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:56,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:56,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:56,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:57,008 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:57,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:59,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:01,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:04,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:04,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:04,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:04,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:05,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:05,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:06,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:06,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:11,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:11,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:13,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:13,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:13,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:13,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:14,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:15,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:16,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:18,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:18,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:18,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:18,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:18,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:19,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:21,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:22,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:22,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:22,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:22,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:22,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:23,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:25,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:26,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:27,007 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:47:27,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:27,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:27,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:27,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:27,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:28,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:29,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:31,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:31,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:31,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:31,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:31,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:33,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:34,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:35,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:35,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:35,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:35,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:36,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:36,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:37,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:39,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:39,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:39,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:39,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:39,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:39,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:41,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:42,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:42,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:42,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:42,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:43,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:44,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:45,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:45,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:45,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:45,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:45,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:46,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:47,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:49,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:49,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:49,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:49,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:49,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:50,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:51,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:51,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:53,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:53,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:53,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:53,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:53,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:55,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:56,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:56,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:56,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:56,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:57,009 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:47:57,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:59,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:01,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:02,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:02,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:02,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:02,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:03,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:05,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:06,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:06,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:06,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:06,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:06,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:06,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:06,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:07,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:11,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:11,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:11,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:11,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:11,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:11,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:12,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:13,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:17,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:18,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:18,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:18,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:18,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:19,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:21,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:21,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:22,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:22,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:22,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:23,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:23,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:26,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:27,011 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:48:28,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:31,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:31,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:31,584 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:31,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:32,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:33,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:33,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:35,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:35,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:35,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:35,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:36,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:36,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:37,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:38,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:39,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:39,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:39,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:39,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:40,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:41,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:41,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:43,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:43,576 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:43,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:43,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:44,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:44,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:45,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:47,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:47,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:47,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:47,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:48,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:49,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:50,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:51,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:53,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:53,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:53,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:53,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:54,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:55,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:55,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:56,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:57,014 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:48:59,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:59,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:59,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:59,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:00,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:01,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:01,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:03,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:03,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:03,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:03,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:03,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:05,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:06,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:07,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:08,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:08,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:08,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:08,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:09,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:09,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:11,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:12,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:12,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:12,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:12,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:12,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:13,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:15,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:17,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:17,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:17,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:17,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:17,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:18,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:19,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:21,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:21,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:21,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:21,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:21,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:22,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:23,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:23,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:25,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:25,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:25,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:25,244 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:25,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:27,016 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:49:27,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:29,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:29,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:29,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:29,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:29,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:30,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:31,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:33,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:33,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:33,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:33,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:34,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:34,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:35,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:35,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:36,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:36,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:36,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:36,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:37,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:39,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:39,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:40,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:41,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:41,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:41,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:41,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:42,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:43,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:45,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:46,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:46,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:46,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:46,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:46,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:47,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:49,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:49,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:49,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:49,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:50,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:50,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:50,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:51,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:54,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:54,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:54,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:54,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:55,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:56,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:57,018 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:49:57,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:57,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:58,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:59,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:01,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:05,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:05,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:05,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:05,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:05,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:06,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:07,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:07,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:09,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:09,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:09,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:09,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:10,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:10,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:11,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:12,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:17,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:18,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:18,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:18,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:18,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:18,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:19,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:20,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:21,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:21,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:21,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:21,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:22,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:23,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:23,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:25,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:25,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:25,052 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:25,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:25,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:25,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:27,020 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:50:27,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:28,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:28,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:28,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:28,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:28,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:29,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:31,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:33,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:34,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:34,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:34,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:34,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:34,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:35,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:35,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:37,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:37,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:37,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:39,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:39,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:40,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:43,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:43,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:43,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:43,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:44,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:44,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:45,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:47,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:47,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:47,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:47,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:48,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:49,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:50,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:50,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:51,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:51,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:51,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:51,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:52,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:53,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:55,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:55,108 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:55,108 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:55,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:55,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:55,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:56,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:57,022 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:50:57,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:59,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:59,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:59,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:59,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:59,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:01,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:02,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:02,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:02,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:02,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:02,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:03,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:05,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:05,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:07,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:07,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:07,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:07,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:07,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:08,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:09,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:10,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:11,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:11,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:11,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:11,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:12,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:13,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:13,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:18,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:19,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:19,727 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:19,727 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:19,732 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:20,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:21,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:23,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:23,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:23,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:23,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:23,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:24,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:25,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:26,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:26,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:26,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:26,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:27,028 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:51:27,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:29,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:30,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:31,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:31,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:31,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:31,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:32,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:33,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:35,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:35,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:36,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:36,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:36,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:36,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:36,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:37,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:40,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:40,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:40,100 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:40,105 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:40,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:40,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:41,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:41,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:43,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:43,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:43,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:43,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:45,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:46,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:47,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:47,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:47,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:47,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:47,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:49,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:50,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:51,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:52,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:52,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:52,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:52,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:52,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:53,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:55,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:57,027 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:51:57,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:58,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:58,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:58,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:58,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:58,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:59,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:02,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:05,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:06,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:06,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:06,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:06,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:07,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:07,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:07,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:09,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:09,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:09,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:10,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:10,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:11,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:12,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:13,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:13,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:13,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:13,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:14,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:15,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:17,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:17,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:17,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:17,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:17,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:18,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:19,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:20,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:22,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:22,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:22,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:22,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:22,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:23,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:24,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:25,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:25,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:25,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:25,860 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:25,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:26,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:27,032 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:52:27,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:30,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:34,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:34,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:34,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:34,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:34,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:35,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:35,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:35,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:38,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:38,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:38,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:38,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:38,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:39,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:40,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:41,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:41,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:41,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:41,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:41,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:42,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:43,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:46,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:50,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:50,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:50,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:50,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:50,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:51,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:52,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:55,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:55,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:55,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:55,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:55,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:57,031 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:52:57,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:58,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:59,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:59,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:59,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:59,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:59,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:01,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:03,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:03,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:03,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:03,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:03,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:04,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:05,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:05,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:07,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:07,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:07,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:07,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:08,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:08,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:09,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:10,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:12,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:12,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:12,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:12,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:12,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:13,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:14,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:19,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:20,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:21,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:21,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:21,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:21,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:21,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:23,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:24,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:25,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:25,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:25,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:25,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:25,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:26,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:27,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:53:27,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:29,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:29,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:29,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:29,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:29,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:30,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:31,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:34,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:34,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:34,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:34,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:35,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:35,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:35,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:35,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:39,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:39,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:39,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:39,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:39,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:40,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:41,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:41,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:42,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:42,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:42,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:42,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:42,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:43,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:46,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:46,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:46,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:46,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:46,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:47,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:49,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:50,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:50,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:50,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:50,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:50,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:51,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:51,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:51,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:55,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:55,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:55,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:55,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:55,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:57,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:53:57,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:57,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:00,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:00,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:00,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:00,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:01,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:02,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:04,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:04,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:04,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:04,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:05,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:05,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:05,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:07,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:09,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:09,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:09,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:09,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:10,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:10,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:11,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:13,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:14,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:14,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:14,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:14,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:14,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:15,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:18,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:18,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:18,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:18,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:18,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:19,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:19,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:20,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:21,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:21,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:21,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:21,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:22,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:23,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:24,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:25,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:27,038 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:54:27,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:27,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:27,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:27,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:28,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:29,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:30,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:31,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:31,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:31,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:31,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:32,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:33,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:35,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:35,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:37,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:37,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:37,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:37,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:37,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:39,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:40,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:41,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:41,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:41,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:41,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:41,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:41,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:43,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:45,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:45,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:45,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:45,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:45,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:47,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:47,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:48,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:48,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:48,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:48,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:48,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:49,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:50,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:51,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:51,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:51,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:51,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:52,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:52,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:53,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:55,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:55,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:55,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:55,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:55,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:55,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:57,040 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:54:57,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:58,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:59,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:59,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:59,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:59,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:00,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:01,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:02,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:02,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:02,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:02,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:02,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:03,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:03,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:05,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:06,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:06,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:06,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:06,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:06,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:07,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:08,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:10,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:11,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:11,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:12,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:13,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:13,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:18,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:18,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:18,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:18,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:18,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:19,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:19,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:20,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:21,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:21,687 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:21,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:21,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:21,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:23,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:24,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:25,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:25,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:25,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:25,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:25,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:25,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:27,042 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:55:27,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:30,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:30,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:30,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:30,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:30,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:34,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:34,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:34,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:34,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:35,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:35,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:35,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:40,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:40,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:43,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:43,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:43,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:43,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:43,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:45,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:46,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:46,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:46,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:46,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:46,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:47,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:50,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:50,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:50,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:50,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:50,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:51,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:51,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:51,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:54,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:54,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:54,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:54,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:54,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:55,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:55,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:57,045 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:55:58,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:58,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:58,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:58,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:59,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:59,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:01,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:02,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:02,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:03,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:05,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:05,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:05,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:05,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:05,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:06,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:07,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:07,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:10,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:10,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:10,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:10,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:10,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:10,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:11,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:13,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:15,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:15,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:15,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:15,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:16,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:17,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:18,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:19,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:19,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:19,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:19,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:19,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:20,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:21,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:22,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:22,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:22,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:22,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:22,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:23,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:24,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:25,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:26,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:26,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:26,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:26,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:26,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:27,047 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:56:27,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:30,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:30,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:30,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:30,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:30,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:30,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:31,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:34,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:34,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:34,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:34,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:34,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:35,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:35,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:35,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:37,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:37,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:37,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:37,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:39,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:40,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:40,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:41,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:41,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:41,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:41,401 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:41,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:43,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:45,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:45,456 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:45,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:45,461 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:45,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:46,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:47,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:48,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:48,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:48,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:48,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:49,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:49,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:50,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:52,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:52,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:52,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:52,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:52,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:52,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:53,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:55,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:57,049 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:56:57,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:58,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:58,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:58,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:58,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:58,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:59,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:02,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:02,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:02,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:02,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:02,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:03,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:03,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:05,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:06,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:06,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:06,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:06,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:06,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:07,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:07,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:10,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:12,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:12,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:12,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:12,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:12,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:12,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:13,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:17,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:17,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:17,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:17,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:17,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:18,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:19,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:20,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:21,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:21,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:21,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:21,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:23,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:23,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:25,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:25,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:25,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:25,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:25,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:25,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:27,051 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:57:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:28,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:28,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:28,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:28,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:28,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:28,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:29,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:33,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:33,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:33,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:33,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:34,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:35,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:35,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:36,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:36,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:36,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:36,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:36,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:37,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:39,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:40,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:40,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:40,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:40,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:40,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:40,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:41,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:45,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:48,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:48,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:48,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:48,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:48,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:49,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:50,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:53,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:53,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:53,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:53,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:53,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:55,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:55,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:56,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:57,053 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:57:57,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:57,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:57,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:59,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:01,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:01,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:01,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:01,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:02,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:02,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:03,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:05,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:07,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:07,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:07,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:07,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:07,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:07,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:07,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:10,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:10,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:10,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:10,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:10,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:10,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:11,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:12,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:16,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:16,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:16,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:16,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:16,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:17,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:18,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:20,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:21,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:21,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:21,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:21,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:22,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:23,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:23,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:25,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:26,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:26,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:26,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:26,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:26,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:27,056 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:58:27,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:29,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:31,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:31,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:31,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:31,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:31,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:31,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:35,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:35,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:35,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:35,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:35,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:35,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:35,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:35,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:40,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:40,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:40,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:40,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:40,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:40,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:40,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:41,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:44,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:44,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:44,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:44,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:44,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:45,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:47,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:47,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:47,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:47,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:47,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:49,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:50,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:50,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:54,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:54,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:54,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:54,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:54,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:55,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:55,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:56,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:57,058 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:58:57,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:57,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:57,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:57,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:57,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:57,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:01,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:02,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:02,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:02,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:02,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:02,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:03,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:06,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:06,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:06,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:06,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:06,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:06,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:07,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:10,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:10,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:10,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:10,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:10,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:11,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:11,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:13,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:14,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:14,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:14,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:14,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:14,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:18,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:21,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:23,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:23,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:23,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:23,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:23,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:23,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:24,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:26,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:27,060 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:59:28,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:28,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:28,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:28,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:29,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:29,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:33,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:33,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:33,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:33,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:33,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:35,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:35,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:36,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:36,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:36,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:36,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:36,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:37,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:37,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:40,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:41,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:41,470 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:41,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:41,475 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:41,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:41,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:43,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:46,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:50,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:50,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:50,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:50,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:50,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:51,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:51,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:52,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:54,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:54,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:54,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:54,513 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:54,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:55,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:56,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:57,065 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:59:58,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:58,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:58,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:58,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:58,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:58,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:59,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:02,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:02,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:02,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:02,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:02,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:03,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:03,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:06,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:08,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:11,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:11,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:11,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:11,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:11,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:12,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:12,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:14,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:17,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:17,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:18,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:18,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:20,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:20,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:20,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:20,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:20,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:21,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:21,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:22,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:24,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:24,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:24,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:24,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:25,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:25,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:26,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:26,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:27,064 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:00:29,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:29,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:29,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:29,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:29,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:30,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:31,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:32,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:32,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:32,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:32,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:33,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:34,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:36,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:36,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:36,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:36,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:36,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:36,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:37,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:38,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:39,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:39,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:39,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:39,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:40,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:41,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:41,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:42,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:43,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:43,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:43,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:43,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:44,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:46,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:46,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:46,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:46,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:46,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:46,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:47,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:48,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:51,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:51,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:51,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:51,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:51,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:51,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:52,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:52,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:54,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:54,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:54,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:54,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:55,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:56,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:56,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:57,070 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:00:57,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:57,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:57,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:58,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:00,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:00,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:00,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:00,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:00,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:01,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:02,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:02,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:05,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:05,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:05,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:05,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:06,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:06,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:06,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:08,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:11,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:11,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:11,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:11,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:11,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:12,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:12,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:14,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:15,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:15,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:15,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:16,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:18,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:19,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:19,384 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:19,384 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:19,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:19,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:20,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:20,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:21,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:23,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:23,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:23,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:23,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:24,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:24,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:26,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:26,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:27,069 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:01:28,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:28,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:28,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:28,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:29,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:29,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:30,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:33,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:33,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:33,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:33,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:34,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:35,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:36,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:36,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:37,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:37,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:37,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:37,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:38,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:38,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:41,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:41,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:41,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:41,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:41,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:41,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:42,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:44,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:46,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:50,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:50,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:50,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:50,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:51,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:51,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:52,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:52,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:55,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:55,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:55,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:55,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:56,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:56,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:57,074 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:01:58,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:58,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:59,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:59,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:59,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:59,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:00,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:00,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:03,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:05,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:05,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:05,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:05,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:05,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:06,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:06,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:09,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:09,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:09,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:09,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:09,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:10,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:10,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:11,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:12,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:12,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:12,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:12,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:13,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:14,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:14,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:19,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:21,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:21,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:21,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:21,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:21,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:22,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:24,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:25,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:26,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:26,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:26,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:26,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:26,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:27,073 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:02:27,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:28,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:30,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:30,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:30,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:31,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:31,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:32,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:35,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:35,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:35,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:35,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:36,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:36,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:37,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:38,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:39,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:39,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:39,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:39,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:40,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:41,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:42,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:42,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:43,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:43,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:43,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:43,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:44,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:46,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:48,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:49,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:49,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:49,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:49,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:49,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:50,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:51,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:53,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:53,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:53,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:53,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:54,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:54,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:54,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:56,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:57,079 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:02:57,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:57,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:57,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:57,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:58,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:58,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:00,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:01,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:01,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:01,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:01,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:02,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:04,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:05,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:05,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:05,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:05,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:06,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:06,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:08,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:10,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:10,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:10,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:10,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:11,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:11,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:11,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:12,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:15,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:15,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:15,740 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:15,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:16,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:16,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:18,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:18,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:18,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:18,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:18,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:19,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:20,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:21,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:21,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:22,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:22,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:22,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:22,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:23,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:24,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:26,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:26,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:26,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:26,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:26,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:26,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:27,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:27,078 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:03:28,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:30,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:30,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:30,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:30,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:30,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:32,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:32,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:33,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:33,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:33,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:33,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:34,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:36,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:36,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:37,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:38,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:38,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:38,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:38,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:38,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:40,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:41,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:41,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:41,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:41,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:41,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:42,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:42,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:43,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:44,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:44,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:44,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:44,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:45,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:46,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:48,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:49,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:49,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:49,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:49,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:50,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:51,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:52,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:54,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:56,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:57,083 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:03:58,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:58,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:58,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:58,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:00,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:00,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:02,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:02,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:02,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:02,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:03,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:04,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:05,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:06,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:06,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:06,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:06,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:06,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:06,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:08,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:11,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:11,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:11,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:11,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:11,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:11,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:15,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:15,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:15,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:15,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:16,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:16,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:20,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:20,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:20,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:20,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:21,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:21,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:21,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:22,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:26,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:26,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:26,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:26,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:26,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:27,082 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:04:27,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:27,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:28,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:30,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:30,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:30,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:30,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:31,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:32,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:32,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:34,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:34,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:34,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:34,263 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:34,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:36,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:37,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:37,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:37,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:37,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:38,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:38,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:40,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:40,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:42,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:42,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:42,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:42,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:43,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:44,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:44,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:47,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:47,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:47,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:47,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:48,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:49,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:50,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:50,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:53,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:53,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:53,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:53,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:54,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:54,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:55,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:55,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:56,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:56,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:56,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:56,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:57,088 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:04:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:58,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:00,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:00,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:00,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:00,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:00,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:01,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:02,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:03,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:03,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:03,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:03,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:04,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:04,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:05,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:07,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:07,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:07,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:08,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:10,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:10,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:10,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:10,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:10,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:11,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:11,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:12,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:14,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:14,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:14,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:14,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:14,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:16,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:17,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:18,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:18,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:18,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:18,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:19,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:20,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:20,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:22,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:22,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:22,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:22,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:23,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:24,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:25,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:25,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:25,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:25,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:25,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:27,087 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:05:28,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:28,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:28,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:28,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:28,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:29,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:30,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:33,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:34,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:34,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:34,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:34,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:35,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:35,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:36,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:38,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:38,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:39,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:39,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:40,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:40,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:43,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:43,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:43,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:43,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:43,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:44,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:45,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:46,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:46,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:46,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:46,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:47,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:48,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:50,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:50,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:51,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:51,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:51,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:51,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:51,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:52,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:55,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:55,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:55,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:55,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:55,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:55,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:56,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:56,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:57,089 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:05:58,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:58,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:59,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:00,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:00,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:01,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:05,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:05,636 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:05,641 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:05,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:06,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:06,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:07,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:08,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:08,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:08,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:08,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:09,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:10,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:10,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:12,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:14,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:14,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:14,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:14,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:15,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:16,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:18,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:18,184 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:18,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:18,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:18,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:18,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:20,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:20,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:21,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:21,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:21,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:21,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:22,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:22,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:23,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:25,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:27,091 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:06:29,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:30,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:30,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:30,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:30,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:30,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:32,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:34,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:34,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:34,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:34,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:34,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:35,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:35,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:36,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:39,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:39,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:39,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:39,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:39,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:40,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:40,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:40,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:44,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:44,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:44,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:44,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:44,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:46,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:46,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:47,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:47,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:47,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:47,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:48,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:48,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:50,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:51,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:52,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:52,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:55,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:56,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:56,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:56,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:56,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:56,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:57,093 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:06:57,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:58,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:00,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:00,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:00,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:00,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:00,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:00,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:02,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:02,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:04,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:04,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:04,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:04,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:04,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:05,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:06,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:07,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:08,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:08,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:08,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:08,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:09,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:10,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:10,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:12,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:13,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:13,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:13,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:13,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:13,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:14,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:17,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:17,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:17,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:17,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:17,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:18,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:18,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:20,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:20,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:20,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:20,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:20,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:21,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:22,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:22,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:25,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:25,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:25,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:25,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:25,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:25,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:26,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:27,095 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:07:28,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:29,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:29,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:29,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:29,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:30,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:30,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:33,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:33,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:33,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:33,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:33,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:34,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:34,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:35,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:37,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:37,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:37,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:37,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:38,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:38,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:38,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:40,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:41,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:41,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:41,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:41,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:41,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:42,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:44,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:44,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:44,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:44,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:44,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:45,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:46,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:48,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:48,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:48,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:48,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:49,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:49,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:50,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:50,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:54,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:55,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:56,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:56,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:56,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:56,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:57,098 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:07:57,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:58,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:00,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:00,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:00,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:00,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:00,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:00,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:02,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:04,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:04,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:04,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:04,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:05,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:05,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:05,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:06,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:08,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:08,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:08,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:08,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:10,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:10,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:11,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:13,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:13,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:13,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:13,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:13,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:14,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:17,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:20,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:21,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:21,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:21,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:21,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:22,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:22,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:22,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:25,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:25,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:25,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:25,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:25,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:26,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:26,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:27,100 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:08:28,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:29,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:29,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:29,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:29,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:29,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:30,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:32,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:32,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:33,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:33,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:34,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:35,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:36,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:36,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:36,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:36,080 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:36,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:38,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:39,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:39,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:39,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:39,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:39,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:40,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:40,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:40,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:43,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:43,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:43,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:43,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:43,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:44,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:44,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:46,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:46,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:46,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:46,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:46,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:48,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:49,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:50,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:50,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:50,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:50,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:50,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:50,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:52,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:53,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:53,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:53,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:53,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:54,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:55,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:57,102 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:08:58,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:58,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:58,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:58,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:58,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:00,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:00,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:02,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:02,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:02,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:02,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:02,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:04,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:05,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:05,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:06,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:06,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:06,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:06,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:06,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:08,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:10,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:10,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:10,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:10,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:10,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:10,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:11,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:12,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:14,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:14,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:14,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:14,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:14,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:16,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:16,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:20,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:20,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:20,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:20,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:20,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:20,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:21,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:22,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:23,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:23,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:23,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:23,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:25,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:26,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:27,105 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:09:27,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:27,694 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:27,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:27,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:31,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:31,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:31,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:31,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:32,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:35,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:35,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:35,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:35,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:35,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:36,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:36,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:37,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:40,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:40,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:40,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:40,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:40,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:41,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:42,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:42,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:44,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:44,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:44,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:44,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:45,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:46,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:48,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:50,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:53,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:53,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:53,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:53,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:53,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:54,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:54,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:55,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:57,107 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:09:58,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:58,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:58,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:58,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:58,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:58,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:00,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:02,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:02,954 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:02,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:02,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:03,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:04,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:05,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:05,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:06,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:06,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:06,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:06,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:06,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:08,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:09,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:09,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:09,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:09,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:10,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:10,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:10,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:11,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:14,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:14,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:14,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:14,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:14,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:16,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:17,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:20,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:22,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:22,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:22,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:22,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:22,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:22,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:24,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:25,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:27,109 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:10:28,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:28,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:28,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:28,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:29,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:32,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:32,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:32,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:32,251 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:32,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:33,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:34,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:35,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:35,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:35,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:35,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:35,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:36,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:36,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:38,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:40,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:40,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:40,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:40,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:40,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:44,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:45,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:45,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:45,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:45,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:46,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:46,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:49,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:49,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:49,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:49,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:49,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:49,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:50,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:50,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:52,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:52,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:52,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:52,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:53,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:54,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:54,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:55,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:57,111 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:00,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:04,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:04,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:04,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:04,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:04,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:05,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:05,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:06,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:09,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:09,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:09,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:09,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:10,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:10,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:10,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:13,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:13,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:13,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:13,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:14,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:14,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:15,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:17,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:17,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:17,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:17,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:17,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:18,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:20,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:21,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:22,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:22,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:22,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:22,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:22,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:24,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:25,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:25,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:25,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:25,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:25,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:26,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:27,116 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:29,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:29,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:29,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:29,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:29,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:30,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:32,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:34,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:34,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:34,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:34,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:35,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:36,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:38,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:39,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:39,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:39,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:39,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:39,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:40,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:40,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:42,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:42,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:42,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:42,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:43,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:43,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:44,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:48,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:48,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:48,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:48,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:48,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:49,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:50,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:50,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:52,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:52,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:52,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:52,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:53,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:54,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:54,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:55,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:56,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:56,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:56,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:56,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:56,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:57,115 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:58,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:00,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:00,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:00,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:00,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:00,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:01,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:02,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:04,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:04,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:04,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:05,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:05,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:06,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:10,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:10,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:13,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:13,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:14,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:16,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:16,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:16,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:16,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:16,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:16,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:18,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:20,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:20,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:20,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:20,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:20,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:20,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:21,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:22,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:24,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:24,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:24,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:24,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:24,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:25,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:26,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:27,121 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:12:27,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:27,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:27,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:27,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:27,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:28,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:28,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:31,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:31,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:31,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:31,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:31,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:32,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:32,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:35,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:35,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:35,112 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:35,117 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:35,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:35,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:36,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:38,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:38,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:38,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:38,796 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:38,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:38,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:40,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:40,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:43,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:44,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:44,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:44,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:44,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:44,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:46,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:48,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:48,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:48,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:48,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:48,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:49,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:50,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:50,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:54,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:55,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:55,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:55,429 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:55,434 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:55,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:55,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:56,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:57,120 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:12:58,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:58,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:58,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:58,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:58,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:59,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:00,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:04,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:04,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:04,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:04,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:04,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:05,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:05,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:06,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:08,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:08,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:08,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:08,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:08,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:10,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:10,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:10,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:12,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:12,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:12,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:12,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:12,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:14,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:16,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:16,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:16,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:16,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:16,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:16,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:18,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:20,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:21,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:24,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:24,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:24,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:24,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:25,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:25,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:26,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:26,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:27,125 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:13:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:32,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:34,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:34,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:34,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:34,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:35,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:36,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:38,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:39,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:39,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:39,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:39,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:40,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:40,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:42,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:43,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:44,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:44,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:44,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:44,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:44,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:46,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:47,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:47,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:47,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:47,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:48,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:48,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:50,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:51,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:52,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:52,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:52,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:52,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:54,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:54,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:56,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:56,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:56,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:56,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:57,125 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:13:58,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:00,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:00,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:00,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:00,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:00,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:00,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:02,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:04,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:04,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:04,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:04,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:04,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:05,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:06,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:06,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:10,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:10,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:10,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:10,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:10,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:11,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:12,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:14,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:14,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:14,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:14,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:14,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:15,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:16,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:17,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:17,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:17,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:17,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:17,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:20,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:20,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:20,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:20,954 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:20,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:20,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:21,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:21,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:22,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:25,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:26,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:27,130 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:14:29,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:29,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:29,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:29,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:29,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:30,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:31,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:33,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:33,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:33,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:33,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:33,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:34,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:36,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:36,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:41,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:41,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:41,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:41,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:41,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:41,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:41,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:42,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:45,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:45,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:45,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:45,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:45,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:45,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:46,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:47,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:49,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:49,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:49,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:49,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:49,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:50,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:51,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:52,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:53,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:53,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:53,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:53,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:53,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:54,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:56,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:56,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:56,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:56,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:56,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:56,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:57,129 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:14:58,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:00,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:00,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:00,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:00,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:00,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:03,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:04,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:04,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:04,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:04,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:04,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:06,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:06,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:08,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:08,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:08,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:08,194 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:09,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:10,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:11,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:11,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:11,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:11,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:11,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:11,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:12,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:14,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:15,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:16,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:16,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:16,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:16,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:18,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:20,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:20,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:20,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:20,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:20,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:20,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:21,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:22,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:25,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:26,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:27,132 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:15:28,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:28,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:28,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:28,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:28,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:30,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:30,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:33,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:33,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:33,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:33,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:34,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:36,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:36,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:36,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:36,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:36,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:36,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:36,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:39,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:40,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:40,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:40,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:40,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:40,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:41,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:42,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:43,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:43,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:43,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:43,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:43,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:44,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:45,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:45,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:46,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:46,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:46,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:46,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:47,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:47,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:49,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:51,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:52,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:54,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:54,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:54,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:54,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:55,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:56,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:57,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:57,134 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:15:58,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:58,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:58,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:58,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:58,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:59,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:00,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:01,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:03,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:05,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:05,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:05,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:05,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:06,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:06,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:07,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:08,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:10,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:10,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:10,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:11,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:13,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:14,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:14,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:14,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:14,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:14,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:15,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:15,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:17,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:18,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:18,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:18,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:18,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:19,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:19,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:21,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:21,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:21,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:21,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:21,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:21,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:22,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:23,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:24,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:25,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:25,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:25,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:25,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:26,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:26,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:27,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:27,136 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:16:29,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:29,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:29,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:29,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:30,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:30,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:31,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:33,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:33,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:33,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:34,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:35,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:35,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:36,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:37,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:37,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:37,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:37,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:38,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:39,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:40,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:40,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:40,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:40,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:40,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:41,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:41,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:43,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:44,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:44,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:44,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:44,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:45,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:46,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:47,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:47,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:47,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:49,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:51,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:51,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:52,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:52,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:52,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:52,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:53,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:55,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:56,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:57,138 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:16:57,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:01,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:01,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:01,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:01,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:02,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:02,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:03,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:05,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:05,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:05,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:05,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:06,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:06,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:07,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:07,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:10,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:10,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:10,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:10,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:11,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:11,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:13,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:13,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:14,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:14,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:14,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:14,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:15,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:19,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:19,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:19,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:19,889 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:19,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:20,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:21,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:21,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:24,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:25,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:25,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:25,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:25,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:26,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:26,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:27,140 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:17:27,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:30,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:33,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:33,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:33,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:33,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:34,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:35,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:35,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:36,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:38,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:38,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:38,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:38,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:39,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:41,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:41,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:41,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:42,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:42,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:42,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:42,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:43,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:45,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:47,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:48,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:48,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:48,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:48,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:49,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:51,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:51,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:52,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:52,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:52,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:52,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:52,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:53,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:53,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:55,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:55,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:55,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:55,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:56,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:56,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:57,142 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:17:57,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:58,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:59,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:59,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:59,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:59,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:00,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:01,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:03,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:07,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:07,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:07,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:07,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:08,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:09,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:09,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:11,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:11,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:11,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:11,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:11,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:12,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:13,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:15,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:15,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:15,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:15,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:16,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:17,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:20,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:20,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:20,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:20,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:20,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:21,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:21,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:23,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:23,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:23,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:24,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:25,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:26,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:26,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:27,144 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:18:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:31,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:31,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:31,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:31,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:31,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:32,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:33,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:35,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:35,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:35,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:35,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:36,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:36,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:37,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:37,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:41,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:41,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:41,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:41,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:41,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:41,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:43,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:43,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:45,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:45,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:45,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:45,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:46,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:47,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:48,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:49,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:49,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:49,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:50,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:51,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:51,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:53,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:53,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:53,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:53,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:54,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:54,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:55,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:56,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:57,149 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:18:58,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:58,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:58,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:58,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:59,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:59,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:01,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:01,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:01,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:01,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:01,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:02,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:03,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:04,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:05,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:06,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:06,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:07,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:10,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:11,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:13,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:13,970 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:13,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:13,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:14,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:15,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:15,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:18,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:18,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:19,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:21,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:21,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:21,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:22,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:22,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:22,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:22,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:25,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:26,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:27,148 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:19:27,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:30,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:30,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:30,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:30,830 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:31,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:32,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:33,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:34,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:34,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:34,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:34,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:35,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:36,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:37,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:37,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:37,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:37,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:37,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:38,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:38,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:39,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:41,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:41,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:41,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:41,466 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:41,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:42,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:43,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:44,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:47,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:47,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:47,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:47,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:48,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:49,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:50,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:51,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:51,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:51,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:51,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:51,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:52,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:53,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:54,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:54,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:54,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:54,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:55,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:55,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:56,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:57,154 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:19:57,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:58,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:58,292 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:58,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:58,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:58,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:01,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:01,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:01,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:01,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:01,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:01,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:02,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:05,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:05,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:05,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:05,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:05,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:06,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:07,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:07,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:10,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:10,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:10,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:10,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:11,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:11,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:12,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:13,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:13,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:13,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:13,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:13,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:14,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:15,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:18,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:18,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:18,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:18,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:18,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:19,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:21,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:21,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:23,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:26,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:26,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:26,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:26,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:26,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:27,153 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:20:27,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:29,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:29,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:30,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:30,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:30,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:30,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:31,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:33,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:33,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:33,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:33,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:33,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:34,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:35,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:36,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:39,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:39,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:39,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:39,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:39,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:40,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:41,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:41,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:42,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:42,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:42,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:42,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:43,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:45,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:45,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:49,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:49,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:49,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:49,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:50,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:51,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:51,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:51,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:53,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:53,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:53,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:53,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:53,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:55,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:56,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:56,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:56,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:56,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:56,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:57,158 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:20:57,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:57,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:59,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:00,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:00,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:00,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:00,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:00,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:01,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:03,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:06,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:06,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:06,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:06,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:06,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:07,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:08,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:09,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:10,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:10,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:10,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:10,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:11,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:11,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:13,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:14,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:16,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:16,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:16,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:16,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:17,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:19,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:19,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:21,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:22,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:22,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:22,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:22,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:22,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:23,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:25,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:26,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:26,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:26,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:26,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:26,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:27,158 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:21:27,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:29,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:30,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:30,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:30,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:30,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:31,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:33,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:34,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:34,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:34,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:34,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:34,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:35,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:35,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:36,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:38,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:38,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:38,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:38,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:39,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:40,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:41,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:41,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:43,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:43,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:43,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:43,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:44,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:45,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:47,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:47,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:47,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:47,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:47,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:49,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:50,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:50,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:50,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:50,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:50,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:51,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:51,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:53,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:53,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:53,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:53,466 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:53,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:53,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:55,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:56,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:56,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:57,163 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:21:59,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:59,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:59,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:59,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:59,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:01,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:02,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:02,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:02,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:02,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:02,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:03,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:05,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:06,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:06,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:06,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:06,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:06,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:06,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:07,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:07,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:11,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:11,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:11,578 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:11,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:11,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:12,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:12,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:13,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:14,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:14,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:14,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:14,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:15,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:17,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:17,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:18,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:18,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:18,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:18,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:19,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:21,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:21,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:22,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:23,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:23,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:23,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:23,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:24,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:25,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:26,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:27,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:27,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:27,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:27,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:27,163 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:22:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:28,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:29,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:32,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:32,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:32,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:32,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:32,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:33,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:34,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:36,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:36,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:36,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:36,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:36,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:37,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:39,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:39,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:40,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:40,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:40,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:41,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:43,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:44,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:44,498 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:44,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:44,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:44,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:45,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:47,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:48,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:48,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:48,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:48,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:48,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:49,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:51,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:51,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:51,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:51,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:51,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:51,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:51,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:53,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:55,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:55,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:55,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:55,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:56,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:56,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:56,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:57,170 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:22:57,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:59,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:59,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:59,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:59,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:00,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:01,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:02,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:02,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:02,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:02,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:02,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:03,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:05,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:06,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:06,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:06,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:06,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:06,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:06,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:07,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:08,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:09,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:09,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:09,999 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:10,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:10,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:11,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:11,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:14,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:15,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:15,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:15,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:16,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:17,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:18,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:18,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:18,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:18,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:19,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:19,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:21,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:21,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:22,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:22,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:22,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:22,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:22,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:25,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:25,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:26,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:27,168 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:23:28,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:28,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:28,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:28,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:29,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:30,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:31,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:33,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:33,085 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:33,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:33,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:33,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:35,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:36,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:37,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:37,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:37,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:37,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:37,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:39,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:41,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:41,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:41,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:41,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:41,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:41,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:41,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:43,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:45,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:45,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:45,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:45,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:46,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:46,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:47,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:49,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:49,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:49,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:49,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:49,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:51,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:51,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:52,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:54,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:54,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:54,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:54,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:54,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:55,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:56,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:57,172 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:23:57,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:57,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:57,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:57,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:57,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:57,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:59,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:02,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:05,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:05,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:05,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:05,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:05,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:06,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:07,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:08,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:09,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:09,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:09,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:09,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:10,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:11,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:11,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:13,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:13,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:13,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:13,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:13,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:15,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:16,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:16,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:16,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:16,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:17,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:18,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:19,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:21,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:23,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:24,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:24,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:24,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:24,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:24,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:25,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:26,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:27,172 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:24:28,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:28,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:28,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:28,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:28,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:29,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:29,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:31,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:31,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:31,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:31,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:32,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:33,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:34,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:35,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:35,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:35,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:35,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:35,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:36,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:37,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:39,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:39,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:39,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:39,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:40,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:41,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:41,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:42,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:42,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:42,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:42,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:42,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:45,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:45,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:45,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:45,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:45,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:45,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:46,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:47,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:50,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:50,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:50,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:50,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:50,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:51,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:51,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:51,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:53,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:53,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:53,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:53,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:55,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:56,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:56,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:56,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:56,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:56,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:56,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:57,177 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:24:57,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:59,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:01,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:01,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:01,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:01,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:01,991 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:02,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:03,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:05,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:05,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:05,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:05,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:05,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:06,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:07,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:07,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:10,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:10,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:10,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:10,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:10,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:11,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:11,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:13,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:15,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:15,054 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:15,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:15,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:15,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:17,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:19,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:19,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:19,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:19,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:19,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:19,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:21,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:21,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:23,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:23,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:23,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:23,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:23,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:24,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:25,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:26,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:26,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:26,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:26,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:26,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:27,177 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:25:27,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:29,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:29,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:32,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:32,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:32,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:33,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:35,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:37,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:37,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:37,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:37,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:37,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:39,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:40,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:43,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:43,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:43,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:43,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:43,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:45,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:46,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:47,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:47,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:47,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:47,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:47,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:49,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:51,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:51,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:53,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:53,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:53,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:53,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:53,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:55,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:56,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:57,181 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:25:57,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:00,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:00,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:00,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:00,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:00,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:01,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:02,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:04,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:04,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:04,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:04,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:06,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:07,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:07,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:08,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:08,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:08,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:08,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:08,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:09,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:11,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:11,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:11,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:11,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:11,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:11,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:13,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:13,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:15,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:15,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:15,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:15,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:15,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:17,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:19,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:21,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:21,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:21,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:21,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:21,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:21,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:23,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:24,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:24,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:24,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:24,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:24,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:25,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:26,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:27,180 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:26:27,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:28,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:28,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:28,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:28,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:28,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:30,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:31,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:33,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:33,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:33,008 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:33,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:33,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:35,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:36,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:36,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:37,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:37,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:37,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:37,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:37,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:39,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:41,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:41,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:41,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:41,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:41,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:41,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:41,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:43,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:45,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:45,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:45,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:45,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:45,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:47,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:47,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:49,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:49,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:49,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:49,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:49,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:51,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:51,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:52,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:53,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:53,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:53,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:53,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:53,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:55,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:56,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:56,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:56,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:56,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:56,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:56,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:57,186 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:26:57,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:58,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:01,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:01,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:01,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:01,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:02,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:03,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:04,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:05,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:05,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:05,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:05,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:05,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:06,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:07,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:09,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:09,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:09,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:09,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:09,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:09,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:11,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:11,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:13,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:13,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:13,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:13,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:13,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:14,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:15,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:15,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:17,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:17,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:17,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:17,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:17,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:19,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:20,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:21,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:22,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:22,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:22,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:22,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:22,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:23,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:25,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:25,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:25,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:25,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:25,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:25,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:26,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:27,185 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:27:27,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:30,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:30,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:30,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:30,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:30,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:31,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:31,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:33,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:33,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:33,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:33,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:33,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:35,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:35,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:36,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:37,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:37,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:37,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:37,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:37,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:39,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:40,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:41,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:42,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:42,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:42,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:42,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:42,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:43,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:47,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:47,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:47,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:47,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:47,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:47,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:49,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:50,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:50,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:50,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:50,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:50,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:50,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:51,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:52,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:54,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:54,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:54,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:54,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:54,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:55,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:55,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:57,190 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:27:57,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:57,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:57,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:57,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:58,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:58,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:59,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:03,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:04,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:04,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:04,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:04,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:04,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:05,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:05,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:08,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:08,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:08,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:08,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:08,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:08,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:09,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:11,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:13,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:15,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:17,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:17,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:17,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:17,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:17,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:18,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:19,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:20,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:20,808 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:20,808 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:20,813 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:20,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:21,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:21,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:23,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:24,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:24,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:24,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:24,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:24,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:26,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:27,190 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:28:27,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:29,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:29,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:29,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:29,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:29,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:29,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:31,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:34,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:34,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:34,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:34,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:34,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:35,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:36,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:40,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:41,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:42,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:42,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:42,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:42,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:42,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:43,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:45,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:45,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:45,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:45,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:45,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:45,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:47,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:49,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:49,681 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:49,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:49,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:49,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:50,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:51,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:51,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:53,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:53,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:53,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:53,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:53,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:55,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:56,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:56,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:56,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:56,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:56,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:56,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:56,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:57,192 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:28:57,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:02,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:03,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:03,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:03,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:03,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:03,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:05,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:06,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:07,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:07,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:07,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:07,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:07,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:07,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:09,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:11,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:11,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:11,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:11,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:11,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:11,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:12,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:13,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:17,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:20,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:20,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:20,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:20,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:20,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:21,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:21,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:23,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:24,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:24,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:24,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:24,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:25,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:26,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:26,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:27,194 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:29:27,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:27,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:27,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:27,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:28,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:29,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:30,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:30,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:30,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:30,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:31,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:32,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:34,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:36,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:37,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:37,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:37,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:37,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:38,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:40,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:40,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:41,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:41,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:41,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:41,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:41,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:42,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:44,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:45,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:45,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:45,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:45,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:46,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:46,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:48,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:50,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:50,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:50,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:50,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:51,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:51,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:51,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:52,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:54,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:54,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:54,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:54,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:55,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:56,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:56,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:57,195 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:29:57,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:58,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:58,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:59,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:00,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:01,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:01,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:01,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:01,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:01,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:03,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:04,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:04,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:04,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:04,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:04,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:05,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:06,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:06,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:08,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:08,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:08,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:08,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:08,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:09,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:10,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:11,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:12,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:12,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:12,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:12,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:13,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:14,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:14,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:16,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:16,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:16,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:16,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:17,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:18,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:19,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:20,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:20,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:20,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:20,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:21,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:21,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:22,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:24,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:24,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:24,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:24,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:24,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:25,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:26,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:26,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:27,200 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:30:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:29,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:29,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:29,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:30,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:32,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:33,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:33,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:33,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:33,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:34,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:35,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:36,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:36,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:37,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:37,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:37,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:37,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:38,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:40,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:41,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:41,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:43,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:43,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:43,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:43,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:44,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:46,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:46,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:47,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:47,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:47,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:47,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:48,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:50,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:51,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:52,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:56,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:56,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:56,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:56,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:56,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:57,199 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:30:58,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:58,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:59,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:00,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:00,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:00,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:00,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:02,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:03,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:03,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:03,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:03,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:04,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:04,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:06,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:06,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:08,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:08,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:08,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:08,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:09,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:09,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:10,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:11,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:12,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:12,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:12,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:12,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:12,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:14,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:15,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:16,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:16,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:16,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:16,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:17,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:18,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:20,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:20,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:20,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:20,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:20,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:21,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:21,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:22,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:24,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:24,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:24,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:24,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:25,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:26,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:26,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:26,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:27,204 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:31:27,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:27,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:27,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:27,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:28,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:30,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:31,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:32,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:32,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:32,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:32,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:32,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:34,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:35,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:35,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:35,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:35,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:36,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:36,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:36,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:38,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:41,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:41,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:43,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:43,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:43,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:43,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:44,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:46,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:47,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:47,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:47,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:47,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:48,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:50,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:51,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:51,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:51,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:51,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:51,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:52,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:52,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:54,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:55,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:55,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:55,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:55,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:56,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:56,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:57,204 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:31:58,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:58,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:59,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:59,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:59,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:59,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:00,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:02,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:03,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:04,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:04,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:04,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:05,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:06,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:06,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:07,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:07,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:07,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:07,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:08,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:08,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:10,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:11,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:13,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:16,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:16,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:16,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:16,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:17,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:18,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:19,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:23,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:23,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:23,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:23,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:24,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:25,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:26,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:26,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:27,209 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:32:27,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:27,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:27,705 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:27,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:28,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:30,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:30,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:31,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:31,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:31,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:31,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:32,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:34,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:35,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:35,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:35,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:35,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:36,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:36,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:36,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:38,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:38,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:38,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:38,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:38,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:39,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:40,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:41,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:41,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:41,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:41,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:44,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:45,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:45,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:45,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:45,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:46,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:47,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:49,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:49,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:49,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:49,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:50,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:51,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:52,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:52,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:54,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:54,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:54,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:54,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:55,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:56,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:56,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:57,208 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:32:58,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:00,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:00,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:00,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:00,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:01,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:02,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:03,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:05,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:06,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:06,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:06,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:06,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:06,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:08,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:09,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:09,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:09,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:09,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:09,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:10,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:10,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:12,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:14,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:15,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:15,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:15,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:15,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:16,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:18,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:19,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:19,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:19,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:19,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:20,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:20,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:20,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:22,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:25,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:25,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:26,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:26,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:26,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:26,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:26,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:27,211 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:33:28,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:31,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:34,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:34,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:34,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:34,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:35,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:35,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:36,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:36,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:37,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:37,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:37,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:37,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:38,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:40,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:40,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:41,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:44,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:44,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:44,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:44,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:45,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:46,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:47,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:48,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:48,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:48,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:48,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:49,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:50,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:50,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:53,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:53,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:53,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:53,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:53,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:54,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:55,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:57,213 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:33:58,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:59,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:59,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:59,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:59,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:59,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:00,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:03,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:04,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:04,350 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:04,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:04,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:05,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:05,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:06,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:08,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:08,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:08,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:08,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:08,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:08,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:10,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:10,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:11,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:11,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:11,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:11,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:13,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:14,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:15,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:15,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:15,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:15,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:15,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:16,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:18,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:18,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:18,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:18,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:19,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:19,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:20,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:20,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:22,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:22,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:22,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:22,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:23,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:24,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:24,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:25,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:25,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:25,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:27,215 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:34:28,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:28,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:28,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:28,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:28,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:29,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:29,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:30,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:33,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:33,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:33,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:33,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:34,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:35,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:35,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:36,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:36,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:36,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:36,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:36,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:37,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:38,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:40,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:40,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:41,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:41,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:41,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:41,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:41,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:42,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:46,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:46,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:46,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:46,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:46,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:47,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:48,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:50,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:50,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:50,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:50,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:50,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:51,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:51,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:52,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:54,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:54,025 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:54,025 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:54,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:54,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:55,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:56,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:57,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:57,217 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:02,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:02,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:02,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:02,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:02,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:04,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:05,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:06,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:06,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:06,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:06,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:06,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:07,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:08,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:10,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:11,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:11,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:11,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:11,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:11,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:12,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:15,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:15,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:15,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:15,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:15,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:16,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:18,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:19,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:19,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:19,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:19,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:20,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:20,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:22,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:23,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:23,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:23,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:23,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:23,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:24,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:25,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:26,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:27,220 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:29,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:29,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:29,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:29,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:29,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:29,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:30,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:33,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:33,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:34,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:34,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:34,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:35,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:36,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:39,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:39,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:39,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:39,029 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:39,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:40,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:40,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:40,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:41,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:41,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:41,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:41,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:42,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:44,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:45,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:45,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:45,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:45,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:45,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:45,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:46,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:48,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:50,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:50,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:50,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:50,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:50,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:51,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:51,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:52,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:54,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:54,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:54,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:54,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:55,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:55,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:56,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:56,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:57,222 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:57,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:57,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:57,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:57,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:58,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:00,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:00,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:01,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:01,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:01,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:01,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:01,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:02,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:02,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:05,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:05,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:05,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:05,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:05,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:06,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:07,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:08,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:08,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:08,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:08,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:08,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:09,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:10,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:10,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:12,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:13,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:13,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:13,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:13,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:14,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:15,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:16,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:17,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:17,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:17,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:17,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:18,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:18,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:20,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:20,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:22,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:22,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:22,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:22,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:22,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:24,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:24,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:25,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:27,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:27,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:27,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:27,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:27,224 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:36:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:28,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:30,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:31,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:31,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:31,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:31,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:32,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:34,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:35,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:35,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:35,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:35,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:35,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:35,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:36,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:38,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:40,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:40,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:41,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:41,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:41,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:41,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:42,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:44,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:45,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:45,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:45,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:45,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:45,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:45,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:46,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:46,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:48,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:48,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:48,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:48,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:49,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:50,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:50,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:52,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:52,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:52,783 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:52,783 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:52,788 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:53,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:54,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:55,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:57,226 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:36:57,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:57,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:57,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:57,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:57,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:00,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:00,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:01,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:01,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:01,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:01,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:01,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:02,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:02,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:04,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:04,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:04,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:04,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:05,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:05,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:06,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:07,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:08,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:08,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:08,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:08,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:09,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:10,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:10,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:12,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:13,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:13,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:13,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:13,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:14,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:15,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:17,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:17,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:17,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:17,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:17,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:18,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:18,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:20,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:20,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:20,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:20,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:20,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:21,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:22,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:23,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:23,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:23,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:23,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:24,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:25,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:26,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:27,228 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:37:27,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:27,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:27,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:27,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:28,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:29,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:30,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:31,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:31,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:31,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:31,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:31,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:32,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:35,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:35,074 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:35,074 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:35,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:35,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:35,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:35,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:36,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:39,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:39,212 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:39,212 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:39,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:39,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:40,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:40,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:43,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:43,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:43,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:43,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:44,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:46,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:47,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:47,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:47,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:47,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:48,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:50,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:51,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:54,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:54,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:54,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:54,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:54,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:55,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:56,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:56,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:57,231 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:37:59,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:59,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:59,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:59,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:59,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:00,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:02,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:02,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:02,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:02,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:02,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:03,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:04,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:05,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:05,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:05,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:05,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:05,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:06,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:07,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:08,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:10,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:10,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:10,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:10,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:10,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:10,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:12,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:13,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:15,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:15,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:15,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:15,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:15,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:16,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:19,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:20,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:23,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:23,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:23,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:23,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:23,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:24,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:24,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:25,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:26,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:26,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:26,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:26,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:27,233 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:38:27,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:28,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:30,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:30,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:30,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:30,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:31,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:32,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:35,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:35,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:39,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:39,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:39,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:39,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:39,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:40,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:40,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:41,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:42,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:42,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:42,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:42,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:43,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:44,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:46,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:46,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:46,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:46,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:46,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:46,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:48,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:49,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:49,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:49,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:49,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:50,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:50,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:50,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:51,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:54,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:54,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:54,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:54,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:54,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:55,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:56,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:57,235 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:38:57,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:58,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:58,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:58,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:58,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:58,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:00,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:01,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:01,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:01,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:01,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:02,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:02,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:04,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:05,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:05,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:05,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:05,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:05,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:06,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:07,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:08,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:09,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:09,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:09,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:09,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:09,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:10,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:10,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:12,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:12,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:12,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:12,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:13,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:13,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:14,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:16,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:16,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:16,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:16,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:16,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:18,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:19,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:20,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:22,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:22,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:22,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:22,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:23,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:24,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:24,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:25,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:26,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,460 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:26,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:26,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:27,237 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:39:28,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:29,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:29,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:29,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:29,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:30,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:30,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:32,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:33,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:33,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:33,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:33,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:33,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:34,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:35,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:36,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:36,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:36,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:36,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:36,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:37,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:40,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:41,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:42,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:42,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:42,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:42,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:42,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:44,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:46,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:46,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:46,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:46,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:46,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:47,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:48,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:50,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:50,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:50,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:50,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:50,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:50,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:52,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:53,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:54,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:54,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:54,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:54,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:55,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:55,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:56,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:57,240 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:39:58,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:58,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:58,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:58,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:58,071 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:58,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:00,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:02,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:02,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:02,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:02,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:03,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:03,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:04,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:05,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:08,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:10,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:11,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:11,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:11,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:11,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:11,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:12,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:14,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:14,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:14,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:14,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:14,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:15,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:16,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:18,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:18,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:18,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:18,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:18,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:19,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:20,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:20,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:23,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:23,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:24,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:25,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:27,242 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:40:28,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:28,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:29,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:30,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:30,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:33,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:33,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:33,086 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:33,091 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:33,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:34,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:35,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:36,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:36,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:36,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:36,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:37,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:38,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:40,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:41,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:41,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:41,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:42,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:44,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:45,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:45,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:45,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:45,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:45,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:46,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:46,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:49,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:49,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:49,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:49,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:49,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:50,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:50,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:52,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:52,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:52,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:52,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:52,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:52,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:54,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:55,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:57,247 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:40:57,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:57,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:57,732 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:57,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:57,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:58,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:58,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:01,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:01,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:01,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:01,410 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:01,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:02,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:03,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:05,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:05,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:05,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:05,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:05,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:05,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:06,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:08,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:09,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:09,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:09,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:09,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:09,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:10,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:10,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:14,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:19,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:20,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:20,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:20,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:20,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:20,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:20,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:22,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:24,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:24,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:24,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:24,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:24,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:25,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:26,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:27,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:27,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:27,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:27,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:27,246 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:41:27,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:28,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:30,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:30,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:30,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:30,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:32,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:35,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:36,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:36,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:36,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:36,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:36,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:38,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:40,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:41,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:44,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:44,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:44,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:44,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:44,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:45,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:46,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:46,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:48,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:48,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:48,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:48,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:48,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:50,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:50,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:52,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:52,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:52,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:52,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:52,190 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:52,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:54,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:55,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:55,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:55,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:55,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:55,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:55,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:56,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:57,251 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:41:57,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:59,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:59,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:59,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:59,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:59,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:02,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:03,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:03,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:03,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:03,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:03,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:04,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:05,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:06,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:06,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:06,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:06,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:06,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:07,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:08,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:10,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:10,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:10,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:10,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:10,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:10,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:12,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:13,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:13,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:13,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:13,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:13,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:14,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:16,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:18,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:18,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:18,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:18,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:18,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:18,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:20,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:20,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:23,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:23,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:23,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:23,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:23,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:23,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:24,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:25,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:27,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:27,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:27,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:27,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,250 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:42:27,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:28,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:29,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:32,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:32,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:32,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:32,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:32,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:34,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:35,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:35,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:38,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:38,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:38,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:38,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:38,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:40,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:40,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:41,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:41,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:41,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:41,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:41,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:42,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:45,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:46,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:49,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:49,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:49,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:50,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:50,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:51,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:52,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:53,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:53,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:53,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:53,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:53,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:54,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:55,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:57,255 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:42:57,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:57,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:57,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:57,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:57,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:57,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:58,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:02,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:03,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:03,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:03,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:03,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:03,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:04,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:05,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:07,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:07,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:07,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:07,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:07,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:07,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:08,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:10,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:12,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:12,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:12,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:12,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:12,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:13,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:14,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:17,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:17,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:17,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:17,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:17,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:18,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:18,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:20,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:22,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:22,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:22,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:22,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:22,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:24,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:24,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:25,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:26,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:26,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:26,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:26,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:27,254 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:43:28,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:29,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:34,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:34,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:34,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:34,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:34,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:34,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:35,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:36,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:39,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:39,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:39,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:39,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:39,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:40,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:40,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:42,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:42,281 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:42,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:42,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:42,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:44,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:45,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:45,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:45,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:45,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:45,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:45,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:46,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:49,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:49,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:49,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:49,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:49,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:50,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:50,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:50,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:53,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:53,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:53,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:53,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:54,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:55,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:55,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:57,259 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:43:57,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:57,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:57,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:57,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:58,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:01,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:01,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:01,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:01,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:01,131 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:01,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:02,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:04,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:04,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:04,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:04,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:04,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:05,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:06,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:06,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:09,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:09,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:09,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:09,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:10,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:10,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:10,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:11,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:15,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:15,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:15,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:15,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:15,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:16,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:17,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:20,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:20,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:20,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:20,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:20,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:20,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:22,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:23,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:23,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:23,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:23,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:23,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:25,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:27,259 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:44:27,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:27,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:27,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:27,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:27,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:28,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:28,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:30,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:30,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:30,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:30,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:30,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:32,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:33,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:34,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:34,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:34,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:34,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:34,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:35,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:36,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:38,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:38,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:38,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:38,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:38,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:39,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:40,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:44,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:46,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:46,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:46,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:46,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:46,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:48,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:49,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:49,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:49,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:49,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:49,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:49,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:50,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:50,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:54,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:54,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:54,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:54,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:54,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:55,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:55,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:56,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:57,264 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:44:58,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:58,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:58,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:58,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:58,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:00,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:00,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:03,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:03,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:03,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:03,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:03,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:04,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:05,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:06,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:07,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:07,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:07,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:07,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:07,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:08,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:10,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:11,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:11,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:11,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:11,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:11,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:12,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:12,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:15,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:15,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:15,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:16,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:17,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:19,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:19,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:19,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:19,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:19,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:20,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:20,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:22,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:23,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:23,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:23,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:23,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:23,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:24,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:25,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:26,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:26,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:26,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:26,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:26,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:27,263 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:45:28,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:29,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:30,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:30,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:30,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:30,194 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:31,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:33,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:34,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:34,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:34,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:34,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:34,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:35,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:35,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:37,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:38,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:38,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:38,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:38,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:39,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:40,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:41,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:42,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:42,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:42,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:42,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:43,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:44,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:45,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:45,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:45,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:45,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:46,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:47,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:49,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:50,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:52,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:52,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:52,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:52,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:53,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:55,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:55,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:55,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:56,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:56,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:56,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:56,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:57,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:57,268 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:45:59,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:00,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:00,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:00,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:00,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:00,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:01,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:03,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:04,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:04,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:04,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:04,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:05,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:05,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:06,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:07,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:10,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:11,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:12,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:12,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:12,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:12,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:13,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:15,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:16,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:16,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:16,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:16,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:16,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:17,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:19,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:19,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:19,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:19,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:19,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:20,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:20,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:21,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:22,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:23,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:23,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:23,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:23,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:24,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:25,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:25,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:27,267 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:46:27,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:27,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:27,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:27,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:27,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:28,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:29,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:32,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:32,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:32,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:32,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:33,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:33,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:35,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:35,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:35,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:35,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:35,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:35,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:36,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:37,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:38,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:39,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:39,212 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:39,212 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:39,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:40,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:40,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:41,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:42,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:42,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:42,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:42,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:43,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:43,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:46,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:46,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:46,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:46,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:47,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:49,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:49,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:50,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:50,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:50,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:50,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:51,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:53,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:54,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:54,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:54,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:54,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:55,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:55,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:55,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:57,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:57,272 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:46:57,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:57,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:57,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:57,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:58,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:59,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:00,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:01,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:01,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:01,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:01,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:02,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:03,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:05,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:05,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:05,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:05,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:05,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:06,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:06,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:07,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:10,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:10,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:10,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:10,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:10,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:11,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:11,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:13,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:13,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:13,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:13,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:13,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:14,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:15,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:16,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:18,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:18,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:18,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:18,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:18,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:19,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:20,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:22,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:22,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:22,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:22,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:22,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:23,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:25,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:25,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:26,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:26,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:26,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:27,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:27,271 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:47:27,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:29,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:31,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:31,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:31,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:31,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:32,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:32,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:35,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:35,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:35,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:35,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:35,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:36,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:37,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:38,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:40,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:40,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:40,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:40,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:40,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:41,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:43,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:44,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:45,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:45,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:45,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:45,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:45,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:47,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:49,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:49,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:49,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:49,258 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:50,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:50,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:50,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:51,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:55,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:57,276 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:47:57,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:57,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:57,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:57,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:58,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:59,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:00,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:05,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:05,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:05,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:05,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:05,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:05,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:06,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:07,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:09,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:09,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:09,382 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:09,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:10,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:10,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:11,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:11,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:12,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:12,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:12,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:12,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:15,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:16,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:16,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:16,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:16,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:16,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:19,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:20,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:21,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:21,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:21,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:21,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:22,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:23,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:25,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:27,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:27,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:27,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:27,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:27,275 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:48:28,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:28,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:29,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:33,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:35,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:35,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:35,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:35,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:35,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:36,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:37,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:38,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:40,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:40,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:40,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:40,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:40,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:41,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:41,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:44,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:44,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:44,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:44,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:44,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:45,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:47,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:47,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:47,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:47,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:47,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:48,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:49,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:50,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:50,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:51,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:51,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:51,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:51,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:52,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:53,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:55,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:55,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:55,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:55,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:55,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:55,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:56,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:57,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:57,278 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:48:59,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:59,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:59,308 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:59,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:00,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:01,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:01,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:02,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:02,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:02,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:02,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:03,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:05,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:05,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:06,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:06,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:06,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:06,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:07,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:07,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:07,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:09,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:09,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:09,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:09,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:10,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:10,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:11,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:12,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:13,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:13,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:13,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:13,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:14,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:15,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:17,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:17,243 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:17,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:17,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:18,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:18,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:19,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:20,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:21,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:21,529 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:21,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:21,534 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:22,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:23,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:23,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:25,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:25,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:25,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:25,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:25,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:26,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:27,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:27,280 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:49:29,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:29,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:29,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:29,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:29,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:30,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:31,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:32,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:32,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:32,971 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:32,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:33,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:35,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:35,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:35,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:35,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:35,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:35,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:35,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:36,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:37,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:39,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:39,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:39,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:39,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:40,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:40,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:40,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:41,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:44,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:44,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:44,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:44,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:45,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:45,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:47,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:50,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:50,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:50,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:50,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:50,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:51,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:51,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:51,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:54,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:54,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:54,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:54,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:55,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:55,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:56,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:57,282 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:49:57,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:59,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:59,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:59,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:59,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:59,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:01,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:02,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:02,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:02,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:02,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:02,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:03,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:05,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:05,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:08,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:10,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:11,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:11,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:11,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:11,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:12,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:13,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:13,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:15,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:15,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:15,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:17,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:19,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:20,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:21,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:21,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:21,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:21,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:21,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:23,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:24,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:24,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:24,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:24,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:24,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:24,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:25,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:25,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:27,284 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:50:27,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:27,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:28,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:29,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:29,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:31,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:31,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:31,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:31,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:31,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:33,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:34,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:34,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:34,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:34,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:35,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:35,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:35,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:37,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:40,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:40,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:43,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:43,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:43,759 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:43,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:44,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:45,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:45,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:48,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:48,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:48,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:48,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:49,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:49,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:50,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:51,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:52,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:52,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:52,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:52,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:53,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:55,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:55,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:56,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:56,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:56,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:56,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:57,286 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:50:57,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:59,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:00,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:00,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:00,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:00,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:01,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:01,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:02,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:05,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:06,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:06,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:06,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:06,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:07,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:07,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:09,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:10,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:10,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:10,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:10,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:10,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:10,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:11,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:13,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:14,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:14,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:14,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:14,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:14,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:15,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:17,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:17,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:17,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:17,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:18,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:18,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:19,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:20,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:22,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:22,727 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:22,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:22,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:23,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:24,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:25,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:25,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:27,288 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:51:30,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:30,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:30,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:30,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:30,900 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:31,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:33,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:34,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:34,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:34,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:34,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:35,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:35,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:35,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:37,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:39,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:39,489 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:39,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:39,494 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:40,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:40,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:41,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:41,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:44,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:44,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:44,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:44,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:45,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:46,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:47,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:50,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:51,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:52,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:52,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:52,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:52,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:52,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:55,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:55,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:55,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:55,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:55,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:56,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:57,290 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:51:57,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:59,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:59,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:59,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:59,731 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:00,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:01,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:02,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:03,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:03,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:03,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:03,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:03,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:05,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:05,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:07,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:07,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:07,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:07,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:08,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:08,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:09,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:10,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:11,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:11,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:11,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:11,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:12,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:13,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:13,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:15,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:15,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:15,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:16,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:18,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:18,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:18,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:18,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:19,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:19,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:20,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:21,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:22,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:22,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:22,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:22,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:23,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:23,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:25,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:25,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:26,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:26,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:26,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:26,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:26,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:27,292 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:52:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:30,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:30,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:30,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:31,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:34,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:34,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:34,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:34,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:35,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:35,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:35,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:37,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:38,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:38,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:38,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:39,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:41,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:42,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:42,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:42,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:43,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:46,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:47,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:47,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:47,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:47,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:48,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:49,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:52,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:54,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:54,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:54,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:54,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:55,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:55,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:56,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:57,294 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:52:57,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:02,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:03,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:03,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:03,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:03,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:04,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:05,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:06,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:07,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:11,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:12,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:12,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:12,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:12,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:13,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:13,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:15,841 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:15,841 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:15,846 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:16,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:17,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:18,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:18,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:18,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:18,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:18,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:19,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:21,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:21,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:23,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:26,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:26,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:26,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:26,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:26,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:27,300 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:53:27,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:29,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:29,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:30,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:30,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:30,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:31,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:33,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:34,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:34,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:34,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:34,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:34,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:35,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:36,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:37,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:38,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:38,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:38,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:38,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:38,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:39,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:40,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:41,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:45,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:46,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:46,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:46,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:46,711 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:47,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:49,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:50,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:50,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:50,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:50,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:50,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:51,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:51,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:51,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:53,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:53,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:53,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:53,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:54,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:55,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:55,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:56,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:57,298 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:53:58,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:58,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:58,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:58,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:58,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:59,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:01,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:02,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:02,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:02,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:02,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:02,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:03,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:05,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:05,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:05,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:05,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:06,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:06,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:06,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:07,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:11,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:11,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:14,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:14,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:14,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:14,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:14,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:15,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:17,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:19,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:19,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:19,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:19,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:19,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:21,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:21,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:22,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:23,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:23,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:23,562 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:23,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:24,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:25,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:26,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:27,303 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:54:28,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:32,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:32,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:32,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:32,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:32,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:33,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:34,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:36,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:36,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:36,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:36,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:36,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:37,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:37,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:39,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:40,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:40,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:40,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:40,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:41,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:41,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:41,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:44,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:44,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:44,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:44,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:44,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:45,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:45,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:48,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:48,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:48,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:48,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:48,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:49,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:51,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:51,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:51,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:51,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:51,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:51,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:52,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:53,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:55,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:55,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:55,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:55,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:55,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:56,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:56,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:57,302 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:54:57,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:00,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:00,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:00,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:00,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:00,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:01,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:02,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:04,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:04,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:04,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:04,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:04,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:05,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:06,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:08,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:09,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:09,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:09,809 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:09,814 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:10,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:11,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:11,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:13,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:18,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:18,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:18,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:18,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:18,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:19,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:21,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:21,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:22,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:22,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:22,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:22,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:22,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:23,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:24,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:25,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:26,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:26,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:27,307 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:55:27,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:29,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:29,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:29,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:29,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:29,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:30,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:31,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:33,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:33,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:33,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:33,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:33,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:34,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:35,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:36,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:39,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:40,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:40,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:40,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:40,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:41,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:41,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:43,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:44,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:45,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:45,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:45,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:46,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:47,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:49,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:49,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:49,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:49,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:49,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:50,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:51,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:51,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:53,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:53,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:53,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:55,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:56,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:56,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:57,306 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:55:57,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:57,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:57,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:57,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:57,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:59,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:01,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:01,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:01,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:01,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:01,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:02,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:03,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:05,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:05,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:05,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:05,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:05,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:06,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:06,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:07,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:09,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:09,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:09,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:09,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:09,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:11,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:11,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:12,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:13,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:13,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:13,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:13,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:13,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:15,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:17,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:21,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:21,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:21,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:21,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:21,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:22,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:22,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:23,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:26,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:27,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:56:28,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:30,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:30,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:30,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:30,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:30,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:31,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:33,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:34,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:34,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:34,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:35,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:36,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:39,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:39,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:39,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:39,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:39,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:39,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:41,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:41,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:42,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:42,828 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:42,828 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:42,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:43,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:43,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:44,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:46,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:46,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:46,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:46,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:46,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:47,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:50,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:51,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:51,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:51,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:51,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:51,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:51,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:53,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:54,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:54,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:54,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:54,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:55,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:55,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:55,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:56,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:57,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:00,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:00,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:00,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:00,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:00,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:01,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:01,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:04,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,384 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:04,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:04,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:05,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:06,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:06,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:07,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:07,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:07,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:07,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:08,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:09,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:11,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:11,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:12,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:12,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:12,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:12,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:13,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:13,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:17,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:17,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:17,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:17,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:17,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:17,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:19,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:21,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:22,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:23,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:23,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:23,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:23,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:23,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:25,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:26,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:27,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:27,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:27,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:27,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:27,316 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:27,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:28,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:29,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:30,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:30,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:30,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:30,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:31,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:31,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:33,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:34,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:34,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:34,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:34,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:35,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:36,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:38,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:38,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:38,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:39,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:39,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:41,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:42,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:42,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:42,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:42,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:43,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:43,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:46,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:46,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:46,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:46,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:46,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:47,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:49,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:50,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:50,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:50,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:50,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:50,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:51,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:51,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:53,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:53,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:53,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:53,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:53,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:55,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:55,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:56,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:57,315 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:57,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:57,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:57,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:57,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:57,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:59,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:00,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:05,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:06,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:06,032 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:06,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:06,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:06,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:06,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:07,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:09,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:09,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:09,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:09,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:09,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:11,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:11,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:11,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:12,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:12,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:12,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:13,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:16,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:17,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:17,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:17,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:17,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:17,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:19,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:21,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:21,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:22,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:22,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:22,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:22,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:22,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:23,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:26,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:26,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:26,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:26,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:26,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:26,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:27,320 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:58:27,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:27,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:30,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:30,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:30,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:30,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:31,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:31,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:32,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:34,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:34,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:34,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:34,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:35,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:35,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:36,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:38,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:39,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:39,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:39,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:39,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:39,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:41,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:41,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:44,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:45,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:45,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:45,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:45,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:45,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:47,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:50,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:51,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:53,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:53,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:53,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:53,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:53,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:55,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:55,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:56,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:57,319 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:58:59,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:59,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:59,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:59,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:59,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:01,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:01,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:03,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:05,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:06,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:07,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:07,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:08,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:09,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:11,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:11,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:11,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:12,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:12,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:13,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:16,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:16,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:16,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:16,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:17,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:17,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:17,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:20,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:20,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:20,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:20,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:20,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:21,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:21,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:23,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:26,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:26,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:26,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:26,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:26,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:26,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:27,321 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:59:27,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:29,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:34,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:34,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:34,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:34,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:34,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:34,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:35,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:36,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:38,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:38,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:38,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:38,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:38,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:39,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:40,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:41,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:41,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:41,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:41,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:41,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:44,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:44,665 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:44,665 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:44,670 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:44,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:45,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:45,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:45,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:48,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:48,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:48,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:48,676 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:48,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:49,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:50,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:51,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:55,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:56,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:57,323 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:00,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:01,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:06,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:06,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:11,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:16,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:16,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:21,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:26,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:27,325 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:29,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:30,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:31,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:36,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:36,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:41,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:46,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:51,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:56,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:57,327 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:59,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:02,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:04,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:07,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:09,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:12,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:14,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:17,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:19,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:22,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:24,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:27,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:27,332 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:01:29,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:32,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:34,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:37,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:39,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:42,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:44,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:47,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:49,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:52,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:57,331 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:01:58,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:59,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:03,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:04,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:08,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:13,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:14,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:18,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:19,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:23,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:24,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:27,333 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:02:28,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:29,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:33,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:34,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:38,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:43,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:44,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:48,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:49,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:53,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:54,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:57,337 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:02:58,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:59,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:03,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:04,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:08,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:13,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:14,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:18,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:19,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:23,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:24,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:27,337 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:03:29,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:29,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:34,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:34,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:39,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:44,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:44,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:49,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:49,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:54,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:54,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:57,339 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:03:59,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:59,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:04:04,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:04,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:04:09,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:14,091 DEBUG SenderThread:452226 [sender.py:send():382] send: telemetry +2024-03-12 21:04:14,091 DEBUG SenderThread:452226 [sender.py:send():382] send: exit +2024-03-12 21:04:14,091 INFO SenderThread:452226 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 21:04:14,091 INFO SenderThread:452226 [sender.py:send_exit():591] handling runtime: 23718 +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:send_exit():597] send defer +2024-03-12 21:04:14,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,096 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 21:04:14,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 1 +2024-03-12 21:04:14,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 21:04:14,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 2 +2024-03-12 21:04:14,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 21:04:14,097 DEBUG SystemMonitor:452226 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 21:04:14,097 DEBUG SystemMonitor:452226 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 21:04:14,100 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 21:04:14,100 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 21:04:14,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined network monitor +2024-03-12 21:04:15,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:15,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:15,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,668 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 21:04:15,668 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 3 +2024-03-12 21:04:15,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,668 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 21:04:15,671 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:15,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 4 +2024-03-12 21:04:15,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,677 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 21:04:15,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,678 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 21:04:15,678 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 5 +2024-03-12 21:04:15,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,678 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 21:04:15,678 DEBUG SenderThread:452226 [sender.py:send():382] send: summary +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:15,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 6 +2024-03-12 21:04:15,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,683 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 21:04:15,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 21:04:15,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:15,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:15,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:16,059 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 7 +2024-03-12 21:04:16,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:16,060 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 21:04:16,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:16,060 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 21:04:16,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:16,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:16,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 21:04:17,399 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 8 +2024-03-12 21:04:17,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:17,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:17,399 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 21:04:17,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:17,400 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 21:04:17,400 INFO SenderThread:452226 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 21:04:17,401 INFO SenderThread:452226 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 21:04:17,491 INFO SenderThread:452226 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 21:04:17,523 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 9 +2024-03-12 21:04:17,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:17,523 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 21:04:17,523 DEBUG SenderThread:452226 [sender.py:send():382] send: artifact +2024-03-12 21:04:17,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:18,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:18,894 INFO wandb-upload_1:452226 [upload_job.py:push():86] Skipped uploading /home/lilei/.local/share/wandb/artifacts/staging/tmpezq3c_ub +2024-03-12 21:04:19,386 INFO wandb-upload_0:452226 [upload_job.py:push():89] Uploaded file /home/lilei/.local/share/wandb/artifacts/staging/tmp7rp7uu2c +2024-03-12 21:04:20,955 INFO SenderThread:452226 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzMTIzODUw', 'versionIndex': 6}}} +2024-03-12 21:04:20,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:20,955 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 21:04:20,955 INFO SenderThread:452226 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 21:04:20,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:21,743 INFO SenderThread:452226 [dir_watcher.py:finish():388] scan: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files +2024-03-12 21:04:21,743 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml config.yaml +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml conda-environment.yaml +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json wandb-metadata.json +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt requirements.txt +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log output.log +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json wandb-summary.json +2024-03-12 21:04:21,744 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 10 +2024-03-12 21:04:21,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:21,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:21,747 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 21:04:21,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:21,753 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 21:04:21,753 INFO SenderThread:452226 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:04:22,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:22,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:22,284 INFO wandb-upload_3:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt +2024-03-12 21:04:22,567 INFO wandb-upload_0:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:22,588 INFO wandb-upload_1:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 21:04:22,603 INFO wandb-upload_2:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 21:04:22,650 INFO wandb-upload_4:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:22,851 INFO Thread-11 (_thread_body):452226 [sender.py:transition_state():617] send defer: 11 +2024-03-12 21:04:22,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:22,851 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 21:04:22,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:22,851 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 21:04:22,851 INFO SenderThread:452226 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 21:04:22,852 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 12 +2024-03-12 21:04:22,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:22,853 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 21:04:22,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:22,853 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 21:04:22,853 INFO SenderThread:452226 [file_stream.py:finish():595] file stream finish called +2024-03-12 21:04:23,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:23,505 INFO SenderThread:452226 [file_stream.py:finish():599] file stream finish is done +2024-03-12 21:04:23,505 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 13 +2024-03-12 21:04:23,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:23,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:23,506 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:23,506 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 21:04:23,506 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 14 +2024-03-12 21:04:23,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:23,506 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send():382] send: final +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send():382] send: footer +2024-03-12 21:04:23,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:23,507 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 21:04:23,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:23,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:23,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:23,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 21:04:23,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: server_info +2024-03-12 21:04:23,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 21:04:23,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 21:04:23,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 21:04:23,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: job_info +2024-03-12 21:04:23,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 21:04:23,699 INFO HandlerThread:452226 [handler.py:finish():869] shutting down handler +2024-03-12 21:04:24,552 INFO WriterThread:452226 [datastore.py:close():296] close: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb +2024-03-12 21:04:24,699 INFO SenderThread:452226 [sender.py:finish():1572] shutting down sender +2024-03-12 21:04:24,699 INFO SenderThread:452226 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:04:24,699 INFO SenderThread:452226 [file_pusher.py:join():178] waiting for file pusher